From 2b326e778c4c76404b75602793f38173527ee1b9 Mon Sep 17 00:00:00 2001 From: jevansnyc Date: Wed, 15 Apr 2026 20:45:20 +0200 Subject: [PATCH 001/395] Add server-side ad templates design spec Co-Authored-By: Claude Sonnet 4.6 --- ...6-04-15-server-side-ad-templates-design.md | 363 ++++++++++++++++++ 1 file changed, 363 insertions(+) create mode 100644 docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md diff --git a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md new file mode 100644 index 000000000..454f37641 --- /dev/null +++ b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md @@ -0,0 +1,363 @@ +# Server-Side Ad Templates Design + +*April 2026* + +--- + +## 1. Problem Statement + +Today's display ad pipeline on most publisher sites is structurally sequential +and browser-bound: + +1. Page HTML arrives at browser +2. Prebid.js (~300KB) downloads and parses +3. Smart Slots SDK scans the DOM to discover ad placements +4. `addAdUnits()` registers slot definitions +5. Prebid auction fires from the browser (~80–150ms RTT to SSPs) +6. Bids return (~1,000–1,500ms window) +7. GPT `setTargeting()` + `refresh()` fires +8. GAM creative renders + +**Total time to ad visible: ~3,100ms.** + +The browser is the slowest possible place to run an auction. It must first download and parse +multiple SDKs, scan the DOM to discover what ad slots exist, and then fire SSP requests over +a consumer internet connection with high and variable latency. + +Trusted Server sits at the Fastly edge — milliseconds from the user, with data-center-to-data-center +RTT to Prebid Server (~20–30ms vs ~80–150ms from a browser). The server knows, from the request +URL alone, exactly which ad slots are available on any given page. There is no reason to wait for +the browser. + +--- + +## 2. Goal + +Enable Trusted Server to: + +1. Match an incoming page request URL against a set of pre-configured slot templates +2. Immediately fire the full server-side auction (all providers: PBS, APS, future wrappers) in + parallel with the origin HTML fetch — before the browser receives a single byte +3. Inject GPT slot definitions into `` so the client can define slots without any SDK +4. Return pre-collected winning bids to the browser's lightweight `/auction` POST before the + browser would have even finished parsing Prebid.js +5. Eliminate Prebid.js from the client entirely + +**Target time to ad visible: ~1,200ms. Net saving: ~2,000ms.** + +--- + +## 3. Non-Goals + +- Eliminating client-side GPT / Google Ad Manager — GAM remains in the rendering pipeline + for Phase 1. The GAM call (`securepubads.g.doubleclick.net`) moves server-side in a future phase. +- Dynamic slot discovery (reading the DOM) — this design commits to pre-defined, URL-matched + slot templates. Smart Slots' dynamic injection behavior is replaced by server knowledge. +- Changing the `AuctionOrchestrator` internally — the orchestrator already handles parallel + provider fan-out. This design adds a new trigger point, not new auction logic. + +--- + +## 4. Architecture + +### 4.1 New File: `creative-opportunities.toml` + +A new config file at the repo root, alongside `trusted-server.toml`. It holds all slot templates: +page pattern matching rules, ad formats, floor prices, and GAM targeting key-values. Bidder-level +params (placement IDs, account IDs) live in Prebid Server stored requests, keyed by slot ID — not +in this file. + +Loaded at build time via `include_str!()`, parsed into `Vec` at startup. +Ad ops can edit this file independently of server configuration. + +`floor_price` is the publisher-owned hard floor per slot — the source of truth for the minimum +acceptable bid price, enforced at the edge before bids reach the ad server. Any bid below the +floor is discarded at the orchestrator level before it enters `__ts_bids`. SSPs may apply their +own dynamic floors independently within their platforms; this floor is the publisher's baseline +that supersedes all other floor logic by virtue of being enforced earliest in the pipeline. + +**Schema:** + +```toml +[[slot]] +id = "atf_sidebar_ad" +page_patterns = ["/20*/"] +formats = [{ width = 300, height = 250 }] +floor_price = 0.50 + +[slot.targeting] +pos = "atf" +zone = "atfSidebar" + +[[slot]] +id = "below-content-ad" +page_patterns = ["/20*/"] +formats = [{ width = 300, height = 250 }, { width = 728, height = 90 }] +floor_price = 0.25 + +[slot.targeting] +pos = "btf" +zone = "belowContent" + +[[slot]] +id = "ad-homepage-0" +page_patterns = ["/", "/index.html"] +formats = [{ width = 970, height = 250 }, { width = 728, height = 90 }] +floor_price = 1.00 + +[slot.targeting] +pos = "atf" +zone = "homepage" +slot_index = "0" +``` + +**Rust type:** + +```rust +#[derive(Debug, Clone, serde::Deserialize)] +pub struct CreativeOpportunitySlot { + pub id: String, + pub page_patterns: Vec, + pub formats: Vec, + pub floor_price: Option, + pub targeting: HashMap, +} +``` + +### 4.2 URL Pattern Matching + +At request time, TS matches the request path against each slot's `page_patterns`. Patterns are +glob-style strings: + +- `/20*/` — matches all date-prefixed article paths (e.g., `/2024/01/my-article/`) +- `/` — matches the homepage exactly +- `/index.html` — exact match + +Multiple slots can match a single URL. All matching slots are collected and fed into a single +auction as separate impressions. Pattern matching is purely in-memory against the pre-parsed +config — sub-millisecond. + +### 4.3 Auction Trigger + +When slots are matched, TS immediately calls `AuctionOrchestrator::run_auction()` with the +matched slots converted to `AdSlot` objects. This happens at request receipt time — in parallel +with the origin fetch. + +The orchestrator's existing behaviour is unchanged: +- All providers (PBS, APS, any configured wrappers) are dispatched simultaneously +- Per-provider timeout budgets are enforced from the remaining auction deadline +- Floor price filtering, bid unification, and winning bid selection are applied as today +- PBS resolves bidder params from its stored requests by slot ID — no bidder params travel + through TS or the browser + +**On NextJS 14 (buffered mode):** TS must buffer the full origin response before forwarding. +This gives the auction the entire origin response time (~150–400ms typical) to run before +any HTML is forwarded. In practice, bids are often collected before origin even responds. + +**On NextJS 16 (streaming mode):** TS streams HTML chunks to the browser immediately. The +auction runs in parallel. Bid injection into `` must complete before the `` tag +is forwarded. If the auction has not returned by the time `` is encountered, TS waits +up to the remaining auction budget, then flushes with whatever bids have arrived (partial +results) or no targeting if timed out. Content after `` is never held. + +### 4.4 Head Injection + +TS injects two separate ``, not +> raw string interpolation. -Prebid.js is eliminated. The client-side ad bootstrap is replaced by a small inline script -(~20 lines) that reads `__ts_ad_slots` and `__ts_bids` and drives GPT directly: +> **Cache contract:** Any response with `__ts_bids` injected is per-user data and must +> not be cached. TS sets `Cache-Control: private, no-store` on the response before +> forwarding, overriding any conflicting cache headers from the publisher origin. +> `Surrogate-Control` and `Fastly-Surrogate-Control` are also stripped. + +### 4.5 Win Notifications + +Win notification responsibilities are split by where the truth lives: + +**`nurl` (SSP win event) — fired server-side.** When the orchestrator selects a winning +bid, TS fires a fire-and-forget background HTTP request to `nurl` from the edge +(edge→SSP RTT ~20–30ms, no auction-path latency cost). A per-integration switch +(`[integrations.prebid].fire_nurl_at_edge`, default `true`) handles cases where the PBS +deployment already fires win events internally to avoid double-firing. APS win +notification follows its own spec. + +**`burl` (billing event) — fired client-side.** `burl` is embedded per slot in +`__ts_bids` (see §4.4). The `__tsAdInit` script registers a GPT `slotRenderEnded` +listener after defining slots. On render: if `!event.isEmpty` and +`event.slot.getTargeting('hb_adid')[0] === bidData.hb_adid`, the client fires `burl` +via `navigator.sendBeacon`. This confirms both that the ad rendered and that our specific +Prebid bid (not a direct deal or backfill) won the GAM line item match. + +### 4.6 Client Residual + +Prebid.js is eliminated. The client-side ad bootstrap is replaced by a small inline +script (~30 lines) that reads `__ts_ad_slots` and `__ts_bids`, drives GPT directly, and +handles billing notifications: ```javascript -window.__tsAdInit = function() { - var slots = window.__ts_ad_slots || []; - var bids = window.__ts_bids || {}; - googletag.cmd.push(function() { - slots.forEach(function(slot) { - var gptSlot = googletag.defineSlot(slot.id, slot.formats, slot.id) - .addService(googletag.pubads()); +window.__tsAdInit = function () { + var slots = window.__ts_ad_slots || [] + var bids = window.__ts_bids || {} + googletag.cmd.push(function () { + slots.forEach(function (slot) { + var gptSlot = googletag + .defineSlot(slot.gam_unit_path, slot.formats, slot.div_id) + .addService(googletag.pubads()) // Apply static targeting from config - Object.entries(slot.targeting).forEach(function([k, v]) { - gptSlot.setTargeting(k, v); - }); + Object.entries(slot.targeting).forEach(function ([k, v]) { + gptSlot.setTargeting(k, v) + }) // Apply pre-won bid targeting if available - var bidTargeting = bids[slot.id] || {}; - Object.entries(bidTargeting).forEach(function([k, v]) { - gptSlot.setTargeting(k, v); - }); - }); - googletag.pubads().enableSingleRequest(); - googletag.enableServices(); - googletag.pubads().refresh(); - }); -}; + var bidData = bids[slot.id] || {} + ;['hb_pb', 'hb_bidder', 'hb_adid'].forEach(function (key) { + if (bidData[key]) gptSlot.setTargeting(key, bidData[key]) + }) + }) + googletag.pubads().enableSingleRequest() + googletag.enableServices() + // Fire burl on confirmed render + googletag.pubads().addEventListener('slotRenderEnded', function (event) { + var slotId = event.slot.getSlotElementId() + var bidData = bids[slotId] || {} + if ( + !event.isEmpty && + bidData.burl && + event.slot.getTargeting('hb_adid')[0] === bidData.hb_adid + ) { + navigator.sendBeacon(bidData.burl) + } + }) + googletag.pubads().refresh() + }) +} ``` -This script is part of the `tsjs-gpt` integration bundle, injected by TS into every matching -page response alongside the existing GPT integration. +This script is part of the existing `gpt` integration bundle +(`crates/js/lib/src/integrations/gpt/index.ts`), extending the existing GPT shim. +Injected via the `gpt` head injector alongside `window.__ts_ad_slots`. --- @@ -238,21 +440,26 @@ t=0ms GET ts.publisher.com/article arrives at Fastly edge t=1ms URL matched against creative-opportunities.toml Slots matched: [atf_sidebar_ad, below-content-ad, section_ad] + Consent check: TCF consent present → auction proceeds t=2ms AuctionOrchestrator.run_auction() called - PBS + APS dispatched in parallel + PBS + APS dispatched in parallel via send_async() Edge→PBS RTT: ~20–30ms -t=2ms Origin fetch dispatched in parallel +t=2ms Origin fetch dispatched via send_async() in parallel + +t=2ms window.__ts_ad_slots script assembled from config (no auction needed) t=150ms Origin HTML arrives at edge (NextJS 14: buffered) + Auction still running; origin response held at edge -t=502ms Auction timeout fires (500ms budget) - Winning bids collected +t=502ms Auction deadline fires (500ms budget) + Winning bids collected; nurl fired as background requests -t=502ms injection assembled: - - window.__ts_ad_slots (from config, available at t=1ms) - - window.__ts_bids (from auction results) +t=502ms HtmlProcessorConfig constructed with bid results captured + injection assembled: + - window.__ts_ad_slots (from config, ready at t=2ms) + - window.__ts_bids (from auction results; Cache-Control: private, no-store set) t=502ms HTML forwarded to browser with injected @@ -270,7 +477,7 @@ t=822ms GET /gampad/ads t=922ms Creative fetch -t=1222ms Creative sub-resources + paint +t=1222ms Creative sub-resources + paint; burl fired via slotRenderEnded AD VISIBLE ~1200ms ``` @@ -279,18 +486,23 @@ t=1222ms Creative sub-resources + paint ## 6. Performance Summary -| Stage | Client-side today | With TS templates | Saving | -|---|---|---|---| -| Script load chain | ~700ms | ~40ms (tsjs only) | -660ms | -| Script parse/JIT | ~280ms | ~10ms | -270ms | -| Sequential SDK hops | ~200ms | 0 | -200ms | -| Auction window | ~1,500ms | ~500ms | -1,000ms | -| GAM + creative | ~570ms | ~570ms | — | -| **Total** | **~3,250ms** | **~1,200ms** | **~2,000ms** | +| Stage | Client-side today | With TS templates | Saving | +| ------------------- | ----------------- | ----------------- | ------------ | +| Script load chain | ~700ms | ~40ms (tsjs only) | -660ms | +| Script parse/JIT | ~280ms | ~10ms | -270ms | +| Sequential SDK hops | ~200ms | 0 | -200ms | +| Auction window | ~1,500ms | ~500ms | -1,000ms | +| GAM + creative | ~570ms | ~570ms | — | +| TTFB penalty¹ | 0 | up to +350ms | - | +| **Total** | **~3,250ms** | **~1,200ms** | **~2,000ms** | + +¹ Buffered mode only: the origin response is held until the auction resolves. For fast +origins (<150ms) and a 500ms auction deadline, TTFB may increase by up to 350ms. This +tradeoff is net-positive on revenue. The streaming mode (NextJS 16) has no TTFB penalty. -Auction RTT improvement: browser fires SSP requests at 80–150ms RTT; edge fires at 20–30ms. -Auction timeout can drop from 1,000–1,500ms to 500ms while still collecting more complete -results, because edge→PBS latency is ~5–7x lower. +Auction RTT improvement: browser fires SSP requests at 80–150ms RTT; edge fires at +20–30ms. Auction timeout can drop from 1,000–1,500ms to 500ms while still collecting +more complete results, because edge→PBS latency is ~5–7x lower. --- @@ -299,24 +511,42 @@ results, because edge→PBS latency is ~5–7x lower. ### New - `creative-opportunities.toml` — slot template config file -- `crates/trusted-server-core/src/creative_opportunities.rs` — config types, TOML parsing, - URL pattern matching, slot-to-`AdSlot` conversion -- `build.rs` update — `include_str!()` for `creative-opportunities.toml` -- Request handler modification — match slots at request receipt, trigger orchestrator immediately, - hold result for head injection -- `tsjs-gpt` integration update — `__tsAdInit` bootstrap replaces Prebid.js ad unit setup +- `crates/trusted-server-core/src/creative_opportunities.rs` — config types, TOML + parsing, URL glob matching, slot-to-`AdSlot` conversion, price bucketing +- `crates/trusted-server-core/build.rs` — `include_str!()` for + `creative-opportunities.toml`; startup slot-ID validation +- `crates/trusted-server-core/src/price_bucket.rs` — Prebid price granularity tables + (dense default; publisher-configurable); converts raw CPM `f64` to `hb_pb` string ### Modified -- `crates/trusted-server-core/src/integrations/prebid.rs` head injector — emit - `window.__ts_ad_slots` from matched slots -- `crates/trusted-server-core/src/html_processor.rs` — inject `window.__ts_bids` once auction - results are available, before `` -- `trusted-server.toml` — add `creative_opportunities_path` config key pointing to the new file +- **`crates/trusted-server-core/src/publisher.rs`** — primary structural change: + - Convert `handle_publisher_request` from `fn` to `async fn` + - Switch origin fetch from `.send()` to `.send_async()` (returns + `PlatformPendingRequest`) + - Add `orchestrator: &AuctionOrchestrator` parameter + - Match slots, check consent, fire auction and origin fetch concurrently + - Await both and construct `HtmlProcessorConfig` with resolved bid results +- **`crates/trusted-server-adapter-fastly/src/main.rs`** — update `route_request` call + site to `.await` the now-async publisher handler; pass orchestrator reference +- **`crates/trusted-server-core/src/html_processor.rs`** — inject `window.__ts_bids` + before `` via `el.on_end_tag()` on the `` element; set + `Cache-Control: private, no-store` header on injection; HTML-escape bid JSON +- **`crates/trusted-server-core/src/integrations/gpt.rs`** — extend head injector to + emit `window.__ts_ad_slots` from matched slots (not `prebid.rs`); emit `__tsAdInit` + bootstrap script +- **`crates/js/lib/src/integrations/gpt/index.ts`** — add `__tsAdInit` function and + `slotRenderEnded` burl-firing logic to the existing GPT shim +- **`crates/trusted-server-core/src/integrations/prebid.rs`** — add + `fire_nurl_at_edge` config key; add nurl fire-and-forget call in orchestrator result + handling +- **`trusted-server.toml`** — add `[creative_opportunities]` section +- **`crates/trusted-server-core/src/settings.rs`** — add `CreativeOpportunitiesConfig` + to `Settings` ### Unchanged -- `AuctionOrchestrator` — no internal changes; new call site only +- `AuctionOrchestrator` internals — no changes; new call site only - PBS stored request configuration — bidder params remain in PBS, keyed by slot ID - GAM line item configuration — targeting key-values pass through unchanged @@ -324,40 +554,66 @@ results, because edge→PBS latency is ~5–7x lower. ## 8. Edge Cases -**No slots match the URL** — auction is not fired. Head injection emits neither global. GPT -bootstrap detects empty `__ts_ad_slots` and skips initialization. Page loads normally with no -ad stack. +**No slots match the URL** — auction is not fired. Neither global is emitted. The page +loads with no TS ad stack; existing client-side Prebid/GPT flow runs unmodified (for +publishers in dual-mode rollout). + +**Consent absent or denied** — auction is not fired. Neither global is emitted. +`Cache-Control: private, no-store` is still set (to prevent caching the consent-negative +response if personalised ads were previously served). Page loads normally; GAM runs its +own auction without Prebid targeting. + +**Auction times out with partial results** — `__ts_bids` is populated with whatever bids +arrived before the deadline. Slots with no bid are omitted. GPT fires without pre-set +targeting for those slots; GAM falls back to its own auction for them. + +**Auction times out with zero results** — `__ts_bids` is an empty object `{}`. All slots +fire GAM without bid targeting. No revenue impact beyond the timeout scenario itself. -**Auction times out with partial results** — `__ts_bids` is populated with whatever bids arrived -before the deadline. Slots with no bid omitted. GPT fires without pre-set targeting for those slots; -GAM falls back to its own auction. +**Origin is slow (NextJS 14, buffered)** — auction has more time; results more likely to +be complete. TTFB impact is bounded by the origin latency, not additive to it. -**Auction times out with zero results** — `__ts_bids` is an empty object `{}`. All slots fire -GAM without bid targeting. No revenue impact beyond the timeout scenario itself (same as today's -fallback). +**NextJS 16 streaming** — `el.on_end_tag()` on `` gates injection. TS waits up to +the remaining `auction_timeout_ms` budget, then flushes. Content after `` is never +held. If the auction resolves before `` is encountered (common case), injection is +zero-latency. -**Origin is slow (NextJS 14, buffered)** — auction has more time; results more likely to be -complete. No change to streaming behavior. +**`creative-opportunities.toml` missing or malformed** — startup fails with a clear +error. No silent degradation. -**NextJS 16 streaming** — TS must flush `` before `` tag passes through. If auction -not yet complete, TS waits up to `auction_timeout_ms` from the config, then flushes. Content -streaming resumes immediately after `` regardless of bid state. +**Config empty (zero slots)** — treated as "no match" for all URLs; auction never fires. +No error. Useful as a kill-switch: deploying an empty `creative-opportunities.toml` +disables the feature without a code change. -**`creative-opportunities.toml` missing or malformed** — startup fails with a clear error. -No silent degradation. +**Slot ID not found in PBS stored requests** — PBS returns a no-bid for that slot. Slot +is omitted from `__ts_bids`. The remaining slots proceed normally. --- ## 9. Open Questions -1. **URL pattern coverage** — does `/20*/` cover all article paths, or are there +1. **URL pattern coverage** — does `/20**` cover all article paths, or are there non-date-prefixed article URLs? Publisher to confirm. 2. **PBS stored request setup** — slot IDs in `creative-opportunities.toml` must have - corresponding stored requests configured in the publisher's PBS instance before this goes live. -3. **Homepage slot count** — the example shows slots 0 and 1. Are there slots 2–5 following - the same pattern? Slot IDs and count to be confirmed with ad ops. -4. **Auction timeout for server-side trigger** — current `[integrations.prebid].timeout_ms` - is 1,000ms. Recommend reducing to 500ms for server-side triggered auctions given the - lower edge→PBS RTT. Separate config key or override on the new trigger path? -5. **`tsjs-gpt` bootstrap delivery** — the `__tsAdInit` script needs to fire after GPT.js - loads. Confirm injection order with the existing GPT integration head injection. + corresponding stored requests configured in the publisher's PBS instance before this + goes live. +3. **Homepage slot count** — the example shows slots 0 and 1. Are there additional slots + following the same pattern? Slot IDs and count to be confirmed with ad ops. +4. **Auction timeout** — ✅ Resolved: new dedicated key + `[creative_opportunities].auction_timeout_ms` with fallback to `[auction].timeout_ms`. + Per-provider ceilings (`[integrations.prebid].timeout_ms`, + `[integrations.aps].timeout_ms`) remain unchanged; the orchestrator's existing + `min(remaining_budget, provider_timeout)` logic applies. +5. **KV-backed config migration path** — Phase 1 ships with `include_str!()` for + simplicity and cost. When ad ops require live slot edits between deploys, the migration + path is: load from `services.kv_store()` at request time with a compiled-in fallback. + Design tracked as a follow-up before Phase 2. +6. **Phase 2 server-side GAM** — The real latency ceiling is the GAM call + (`securepubads.g.doubleclick.net`). Phase 2 routes the GAM ad request through the edge + (securepubads proxy + creative bundling), eliminating the last browser→Google hop. The + Phase 1 architecture is designed to be shape-compatible with this: `__ts_ad_slots` + gives the edge the full slot inventory it needs to build a server-side GAM request. +7. **`tsjs-gpt` bootstrap delivery** — ✅ Resolved: `__tsAdInit` is part of the existing + `gpt` integration bundle, not a new integration. Injection order: `window.__ts_ad_slots` + → existing GPT shim → `__tsAdInit` — all emitted by the `gpt` head injector in a single + `".to_string() + ), + ad_bids_script: None, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"T", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!(html.contains("window.__ts_ad_slots"), "should inject ad slots"); + } + + #[test] + fn injects_bids_before_end_of_head() { + let bids_script = ""; + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_script: Some(bids_script.to_string()), + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"T", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!(html.contains("window.__ts_bids"), "should inject bids"); + let bids_pos = html.find("window.__ts_bids").expect("should find bids"); + let end_head_pos = html.find("").expect("should find "); + assert!(bids_pos < end_head_pos, "bids script should appear before "); + } + ``` + + Run: `cargo test -p trusted-server-core html_processor` + Expected: compile error (no `ad_slots_script`/`ad_bids_script` fields, no `empty_for_tests()`) + +- [ ] **Step 2: Add `empty_for_tests()` to `IntegrationRegistry`** + + In `registry.rs`, add: + + ```rust + #[cfg(test)] + impl IntegrationRegistry { + pub fn empty_for_tests() -> Self { + // Minimal registry with no integrations for unit testing html_processor + Self { + inner: Arc::new(RegistryInner { + proxies: Default::default(), + attribute_rewriters: Default::default(), + script_rewriters: Vec::new(), + html_post_processors: Vec::new(), + head_injectors: Vec::new(), + metadata: Default::default(), + }) + } + } + } + ``` + + (Adjust field names to match the actual `RegistryInner` struct.) + +- [ ] **Step 3: Add fields to `HtmlProcessorConfig`** + + ```rust + pub struct HtmlProcessorConfig { + pub origin_host: String, + pub request_host: String, + pub request_scheme: String, + pub integrations: IntegrationRegistry, + /// Pre-computed `` for matched slots. + /// Injected at open, before integration head inserts. `None` when no slots matched. + pub ad_slots_script: Option, + /// Pre-computed `` for winning bids. + /// Injected immediately before via on_end_tag(). `None` when auction not run. + pub ad_bids_script: Option, + } + ``` + + Update `from_settings` to initialize `ad_slots_script: None, ad_bids_script: None`. + +- [ ] **Step 4: Inject `__ts_ad_slots` at head-open AND register `on_end_tag` for `__ts_bids`** + + In `create_html_processor`, within the EXISTING single `element!("head", ...)` handler, make two changes: + 1. Prepend the ad slots script BEFORE the existing integration inserts: + + ```rust + // NEW: inject __ts_ad_slots first + if let Some(ref slots_script) = ad_slots_script { + snippet.push_str(slots_script); + } + // ... existing: for insert in integrations.head_inserts(&ctx) { ... } + ``` + + 2. After `el.prepend(...)`, register the end-tag handler for `__ts_bids`: + ```rust + // Register on_end_tag handler for __ts_bids injection before + if let Some(bids_script) = ad_bids_script.clone() { + el.on_end_tag(move |end_tag| { + end_tag.before(&bids_script, ContentType::Html); + Ok(()) + })?; + } + ``` + + Both changes live inside the same `element!("head", ...)` closure — no second handler needed. + + Capture `ad_slots_script` and `ad_bids_script` into the closure the same way as `injected_tsjs`: + + ```rust + let ad_slots_script = config.ad_slots_script.clone(); + let ad_bids_script = config.ad_bids_script.clone(); + ``` + + > **lol_html `on_end_tag` API note:** `Element::on_end_tag(handler)` is available in lol_html ≥2.0. The handler receives `&mut EndTag` and must return `Result<(), Box>`. Use `ContentType::Html` so the injected `", escaped) + } + + pub(crate) fn build_ad_bids_script( + winning_bids: &std::collections::HashMap, + price_granularity: crate::price_bucket::PriceGranularity, + ) -> String { + let bids_map: serde_json::Map = winning_bids + .iter() + .filter_map(|(slot_id, bid)| { + let cpm = bid.price?; + let entry = serde_json::json!({ + "hb_pb": price_bucket(cpm, price_granularity), + "hb_bidder": bid.bidder, + "hb_adid": bid.ad_id.as_deref().unwrap_or(""), + "burl": bid.burl, + }); + Some((slot_id.clone(), entry)) + }) + .collect(); + let json = serde_json::to_string(&serde_json::Value::Object(bids_map)) + .expect("should serialize bids"); + let escaped = html_escape_for_script(&json); + format!("", escaped) + } + + /// HTML-escape a JSON string for safe inline `" + .to_string(), + // __tsAdInit definition — reads window.__ts_ad_slots / __ts_bids at call time. + concat!( + "" + ).to_string(), + ] + } + } + ``` + +- [ ] **Step 3: Run tests** + + Run: `cargo test -p trusted-server-core integrations::gpt` + Expected: all pass including new test + +- [ ] **Step 4: Commit** + + ```bash + git add crates/trusted-server-core/src/integrations/gpt.rs + git commit -m "Emit __tsAdInit function definition from GPT head injector" + ``` + +--- + +## Task 10: `gpt/index.ts` — TypeScript `__tsAdInit` + +**Files:** + +- Modify: `crates/js/lib/src/integrations/gpt/index.ts` + +The TypeScript version is the authoritative implementation; it must mirror the Rust inline string from Task 9 exactly. + +- [ ] **Step 1: Write a failing test** + + In `crates/js/lib/src/integrations/gpt/index.test.ts`: + + ```typescript + import { describe, it, expect, vi, beforeEach } from 'vitest' + + describe('installTsAdInit', () => { + beforeEach(() => { + delete (window as any).__ts_ad_slots + delete (window as any).__ts_bids + delete (window as any).__tsAdInit + }) + + it('defines googletag slots from __ts_ad_slots and calls refresh', () => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + } + const mockPubads = { + enableSingleRequest: vi.fn(), + addEventListener: vi.fn(), + refresh: vi.fn(), + getTargeting: vi.fn().mockReturnValue([]), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [ + { + id: 'atf', + gam_unit_path: '/123/atf', + div_id: 'atf', + formats: [[300, 250]], + targeting: { pos: 'atf' }, + }, + ] + ;(window as any).__ts_bids = { + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + burl: 'https://ssp/bill', + }, + } + + // Must import installTsAdInit from the module + const { installTsAdInit } = require('./index') + installTsAdInit() + ;(window as any).__tsAdInit() + + expect((window as any).googletag.defineSlot).toHaveBeenCalledWith( + '/123/atf', + [[300, 250]], + 'atf' + ) + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00') + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'kargo') + expect(mockPubads.refresh).toHaveBeenCalled() + }) + + it('fires burl via sendBeacon on slotRenderEnded when our bid won', () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) + // ... setup and trigger slotRenderEnded event + // Verify: navigator.sendBeacon called with burl + beaconSpy.mockRestore() + }) + }) + ``` + + Run: `cd crates/js/lib && npx vitest run` + Expected: FAIL — `installTsAdInit` not exported + +- [ ] **Step 2: Add `installTsAdInit` to `index.ts`** + + Add to `crates/js/lib/src/integrations/gpt/index.ts` (bottom of file): + + ```typescript + interface TsAdSlot { + id: string + gam_unit_path: string + div_id: string + formats: Array + targeting: Record + } + + interface TsBidData { + hb_pb?: string + hb_bidder?: string + hb_adid?: string + burl?: string + } + + type TsWindow = Window & { + __ts_ad_slots?: TsAdSlot[] + __ts_bids?: Record + __tsAdInit?: () => void + } + + /** + * Install `window.__tsAdInit` — reads `window.__ts_ad_slots` and `window.__ts_bids` + * (injected by the edge into ), defines GPT slots, applies pre-won bid targeting, + * registers a `slotRenderEnded` listener to fire `burl` via `sendBeacon`, then calls + * `refresh()`. + */ + export function installTsAdInit(): void { + const w = window as TsWindow + w.__tsAdInit = function () { + const slots = w.__ts_ad_slots ?? [] + const bids = w.__ts_bids ?? {} + const g = (window as GptWindow).googletag + if (!g) return + g.cmd.push(() => { + slots.forEach((slot) => { + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats, + slot.div_id + ) + if (!gptSlot) return + gptSlot.addService(g.pubads()) + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => + gptSlot.setTargeting(k, v) + ) + const bid = bids[slot.id] ?? {} + ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + }) + }) + g.pubads().enableSingleRequest() + g.enableServices() + g.pubads().addEventListener?.('slotRenderEnded', (event: any) => { + const slotId: string = event.slot?.getSlotElementId?.() ?? '' + const bid = bids[slotId] ?? {} + if ( + !event.isEmpty && + bid.burl && + event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + ) { + navigator.sendBeacon(bid.burl) + } + }) + g.pubads().refresh() + }) + } + } + ``` + + Call `installTsAdInit()` from the integration's initialization path so it's set up when the bundle loads. + +- [ ] **Step 3: Run JS tests** + + Run: `cd crates/js/lib && npx vitest run` + Expected: new tests pass + +- [ ] **Step 4: Build JS bundle** + + Run: `cd crates/js/lib && node build-all.mjs` + Expected: clean build + +- [ ] **Step 5: Commit** + + ```bash + git add crates/js/lib/src/integrations/gpt/ + git commit -m "Add __tsAdInit and slotRenderEnded burl firing to GPT integration" + ``` + +--- + +## Task 11: `nurl` fire-and-forget + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/prebid.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Write failing test** + + ```rust + #[test] + fn prebid_config_fire_nurl_defaults_to_true() { + let config = PrebidConfig::default(); + assert!(config.fire_nurl_at_edge, "should fire nurl at edge by default"); + } + ``` + + Run: `cargo test -p trusted-server-core integrations::prebid` + Expected: FAIL + +- [ ] **Step 2: Add `fire_nurl_at_edge` to `PrebidConfig`** + + ```rust + #[serde(default = "default_fire_nurl_at_edge")] + pub fire_nurl_at_edge: bool, + ``` + + ```rust + fn default_fire_nurl_at_edge() -> bool { true } + ``` + +- [ ] **Step 3: Fire nurls in publisher.rs after auction** + + After `auction_result` is obtained, add: + + ```rust + if let Some(ref result) = auction_result { + fire_winning_nurls(result, settings); + } + ``` + + Add helper (no `.await` — fire-and-forget): + + ```rust + fn fire_winning_nurls( + result: &crate::auction::orchestrator::OrchestrationResult, + settings: &Settings, + ) { + use crate::backend::BackendConfig; + + let fire_nurl = settings + .integrations + .get_typed::("prebid") + .map(|c| c.fire_nurl_at_edge) + .unwrap_or(true); + + if !fire_nurl { + return; + } + + for bid in result.winning_bids.values() { + let Some(ref nurl) = bid.nurl else { continue }; + let backend_name = match BackendConfig::from_url(nurl, false) { + Ok(name) => name, + Err(e) => { + log::warn!("nurl: cannot create backend for {nurl}: {e:?}"); + continue; + } + }; + match fastly::Request::get(nurl).send_async(&backend_name) { + Ok(_) => log::debug!("nurl: fired for slot {}", bid.slot_id), + Err(e) => log::warn!("nurl: failed for slot {}: {e}", bid.slot_id), + } + } + } + ``` + +- [ ] **Step 4: Run tests** + + Run: `cargo test --workspace` + Expected: all pass + +- [ ] **Step 5: Commit** + + ```bash + git add crates/trusted-server-core/src/integrations/prebid.rs \ + crates/trusted-server-core/src/publisher.rs + git commit -m "Fire winning bid nurl fire-and-forget from edge; add fire_nurl_at_edge config" + ``` + +--- + +## Task 12: End-to-end integration tests + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` (test module) + +Tests use `pub(crate)` helpers from Task 8 directly. + +- [ ] **Step 1: Write tests** + + In `publisher.rs` test module: + + ```rust + #[cfg(test)] + mod creative_opportunities_tests { + use super::{build_ad_slots_script, build_ad_bids_script, html_escape_for_script}; + use crate::creative_opportunities::{ + CreativeOpportunitiesConfig, CreativeOpportunitySlot, CreativeOpportunityFormat, + CreativeOpportunitiesFile, match_slots, + }; + use crate::auction::types::{Bid, MediaType}; + use crate::price_bucket::PriceGranularity; + use std::collections::HashMap; + + fn make_config() -> CreativeOpportunitiesConfig { + CreativeOpportunitiesConfig { + gam_network_id: "21765378893".to_string(), + auction_timeout_ms: Some(500), + price_granularity: PriceGranularity::Dense, + } + } + + fn make_slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: "atf_sidebar_ad".to_string(), + gam_unit_path: Some("/21765378893/publisher/atf-sidebar".to_string()), + div_id: Some("div-atf-sidebar".to_string()), + page_patterns: vec!["/20**".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, height: 250, media_type: MediaType::Banner, + }], + floor_price: Some(0.50), + targeting: [("pos".to_string(), "atf".to_string())].into_iter().collect(), + providers: Default::default(), + } + } + + #[test] + fn ad_slots_script_is_safe_and_parseable() { + let slots = vec![make_slot()]; + let config = make_config(); + let script = build_ad_slots_script(&slots, &config); + assert!(script.contains("window.__ts_ad_slots=JSON.parse"), "should use JSON.parse"); + assert!(script.contains("atf_sidebar_ad"), "should include slot id"); + // Verify no raw < or > that could break HTML parser + let inner = script.trim_start_matches(""); + assert!(!inner.contains('<'), "no unescaped < in script content"); + assert!(!inner.contains('>'), "no unescaped > in script content"); + } + + #[test] + fn ad_bids_script_uses_price_bucket_and_ad_id() { + let mut winning_bids = HashMap::new(); + winning_bids.insert("atf_sidebar_ad".to_string(), Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(2.53), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, height: 250, + nurl: None, + burl: Some("https://ssp.example/billing?id=abc123".to_string()), + ad_id: Some("prebid-uuid-abc123".to_string()), + metadata: HashMap::new(), + }); + let script = build_ad_bids_script(&winning_bids, PriceGranularity::Dense); + assert!(script.contains("\"hb_pb\":\"2.53\""), "should bucket 2.53 as 2.53 (dense)"); + assert!(script.contains("\"hb_bidder\":\"kargo\""), "should include bidder"); + assert!(script.contains("\"hb_adid\":\"prebid-uuid-abc123\""), "should use ad_id not creative markup"); + assert!(script.contains("burl"), "should include burl for billing"); + } + + #[test] + fn html_escape_neutralizes_xss_in_json() { + let malicious = r#"{"zone":""), "should escape "); + assert!(escaped.contains("\\u003c"), "should unicode-escape <"); + assert!(escaped.contains("\\u003e"), "should unicode-escape >"); + } + + #[test] + fn url_matching_end_to_end() { + let file = CreativeOpportunitiesFile { slots: vec![make_slot()] }; + assert_eq!(match_slots(&file.slots, "/2024/01/my-article").len(), 1, "should match article"); + assert_eq!(match_slots(&file.slots, "/about").len(), 0, "should not match /about"); + assert_eq!(match_slots(&file.slots, "/").len(), 0, "should not match root"); + } + } + ``` + +- [ ] **Step 2: Run tests** + + Run: `cargo test -p trusted-server-core creative_opportunities_tests` + Expected: all pass + +- [ ] **Step 3: Run full suite + CI gates** + + ```bash + cargo test --workspace + cargo clippy --workspace --all-targets --all-features -- -D warnings + cargo fmt --all -- --check + cd crates/js/lib && npx vitest run + cd crates/js/lib && npm run format + cd docs && npm run format + ``` + + Expected: all clean + +- [ ] **Step 4: Commit** + + ```bash + git add crates/trusted-server-core/src/publisher.rs + git commit -m "Add integration tests for creative opportunities pipeline (slots, bids, XSS)" + ``` + +--- + +## Manual Verification Checklist + +Run `fastly compute serve` and verify: + +- [ ] **No match:** Request `/about` — no `__ts_ad_slots` or `__ts_bids` in response HTML, no `Cache-Control: private, no-store` +- [ ] **Match:** Request `/2024/01/article` — both globals present in ``, `Cache-Control: private, no-store` set +- [ ] **Empty file kill-switch:** Empty `creative-opportunities.toml` → no globals injected on any URL +- [ ] **Auction timeout:** Set `auction_timeout_ms = 1` → `__ts_bids` injects as `{}`, no slot entries +- [ ] **XSS check:** Add `targeting = { zone = " +``` + +> **Security:** All string values are JSON-serialized via `serde_json` and HTML-escaped +> before insertion into the ``, not -> raw string interpolation. +- If the auction has already completed for ``, response returns immediately + with cached results (cache hit). Typical case for non-trivial origin times. +- If the auction is still in flight, the request blocks until completion or `A_deadline`, + whichever fires first. Long-poll semantics, capped by the auction timeout. +- If `` is unknown (cache miss, expired TTL, or never created), returns + `404`. Client falls back to firing GPT without pre-set targeting. +- If no slot received a bid above floor, returns `{}`. Client fires GPT without targeting. +- Response carries `Cache-Control: private, no-store`. -> **Cache contract:** Any response with `__ts_bids` injected is per-user data and must -> not be cached. TS sets `Cache-Control: private, no-store` on the response before -> forwarding, overriding any conflicting cache headers from the publisher origin. -> `Surrogate-Control` and `Fastly-Surrogate-Control` are also stripped. +**Storage:** auction results cached in-process (per-edge-instance) keyed by request ID +with a 30-second TTL. Sized small (a few KB per entry) and short-lived; no Fastly KV +write on the hot path. + +**Security:** request IDs are 128-bit unguessable UUIDs. Even if a request ID leaks, the +worst-case impact is reading bid metadata that's already destined for that session's +GPT slots — no cross-user data exposure. ### 4.5 Win Notifications @@ -386,119 +455,357 @@ Prebid bid (not a direct deal or backfill) won the GAM line item match. ### 4.6 Client Residual Prebid.js is eliminated. The client-side ad bootstrap is replaced by a small inline -script (~30 lines) that reads `__ts_ad_slots` and `__ts_bids`, drives GPT directly, and -handles billing notifications: +script that reads `__ts_ad_slots`, fetches bids from `/ts-bids`, drives GPT directly, +and handles billing notifications. Slot definition happens immediately; bid targeting +and `refresh()` happen after `/ts-bids` resolves: ```javascript window.__tsAdInit = function () { var slots = window.__ts_ad_slots || [] - var bids = window.__ts_bids || {} + var rid = window.__ts_request_id + + // Kick off bid fetch as early as possible. Fires in parallel with GPT setup. + var bidsPromise = rid + ? fetch('/ts-bids?rid=' + encodeURIComponent(rid), { credentials: 'omit' }) + .then(function (r) { + return r.ok ? r.json() : {} + }) + .catch(function () { + return {} + }) + : Promise.resolve({}) + googletag.cmd.push(function () { - slots.forEach(function (slot) { + // Define slots immediately — no auction wait + var gptSlots = slots.map(function (slot) { var gptSlot = googletag .defineSlot(slot.gam_unit_path, slot.formats, slot.div_id) .addService(googletag.pubads()) - // Apply static targeting from config Object.entries(slot.targeting).forEach(function ([k, v]) { gptSlot.setTargeting(k, v) }) - // Apply pre-won bid targeting if available - var bidData = bids[slot.id] || {} - ;['hb_pb', 'hb_bidder', 'hb_adid'].forEach(function (key) { - if (bidData[key]) gptSlot.setTargeting(key, bidData[key]) - }) + return { id: slot.id, gptSlot: gptSlot } }) + googletag.pubads().enableSingleRequest() googletag.enableServices() - // Fire burl on confirmed render - googletag.pubads().addEventListener('slotRenderEnded', function (event) { - var slotId = event.slot.getSlotElementId() - var bidData = bids[slotId] || {} - if ( - !event.isEmpty && - bidData.burl && - event.slot.getTargeting('hb_adid')[0] === bidData.hb_adid - ) { - navigator.sendBeacon(bidData.burl) - } + + // Apply bid targeting and refresh once /ts-bids resolves. + bidsPromise.then(function (bids) { + gptSlots.forEach(function ({ id, gptSlot }) { + var bidData = bids[id] || {} + ;['hb_pb', 'hb_bidder', 'hb_adid'].forEach(function (key) { + if (bidData[key]) gptSlot.setTargeting(key, bidData[key]) + }) + }) + + // Fire burl on confirmed render + googletag.pubads().addEventListener('slotRenderEnded', function (event) { + var slotId = event.slot.getSlotElementId() + var bidData = bids[slotId] || {} + if ( + !event.isEmpty && + bidData.burl && + event.slot.getTargeting('hb_adid')[0] === bidData.hb_adid + ) { + navigator.sendBeacon(bidData.burl) + } + }) + + googletag.pubads().refresh() }) - googletag.pubads().refresh() }) } ``` +**Why slot definition happens before bid fetch resolves:** GPT slot definition is +synchronous and cheap. Defining slots early lets GPT prepare iframes and start any +internal work that doesn't require ad server response. `refresh()` is the call that +actually triggers the GAM ad request — that's the one we delay until bids arrive. + +**Failure modes:** + +- `/ts-bids` returns 404 (unknown rid, TTL expired) → `bidsPromise` resolves to `{}`, + `refresh()` fires without bid targeting, GAM falls back to its own auction. Same + graceful degradation as no-bid case. +- `/ts-bids` network failure → caught, resolves to `{}`, same fallback. +- Auction times out server-side → `/ts-bids` returns `{}`, same fallback. + This script is part of the existing `gpt` integration bundle (`crates/js/lib/src/integrations/gpt/index.ts`), extending the existing GPT shim. Injected via the `gpt` head injector alongside `window.__ts_ad_slots`. +### 4.7 Caching Behavior + +Page assets and bid results have very different cacheability properties. The +architecture is designed so that everything that can be cached, is. + +**What gets cached where:** + +| Asset | Cached at | Cacheability | +| ------------------------ | -------------------------------- | --------------------------------------------------------- | +| Origin HTML | Fastly edge HTTP cache | Yes, if origin sends `Cache-Control: public, max-age=...` | +| Origin CSS / fonts / JS | Fastly edge + browser | Yes (typically hashed URLs, immutable) | +| `tsjs` bundle | Fastly edge + browser | Yes (already content-hashed via `bundle.rs`, immutable) | +| `__ts_ad_slots` payload | Could be precomputed per pattern | In-memory match is sub-millisecond — not worth caching | +| `__ts_request_id` | **Never** | Per-request UUID, minted at request receipt | +| Bid results (`/ts-bids`) | In-process `bid_cache`, 30s TTL | Per-request, never shared across users | + +**Architecture:** + +1. Fastly's built-in HTTP cache stores the **origin response** keyed by URL. TS + does not implement its own HTML caching layer — it leverages the existing + Fastly cache. +2. On request: TS reads from cache (cache hit, ~5ms) or fetches from origin + (cache miss, ~150ms typical). +3. TS injects `__ts_ad_slots` + `__ts_request_id` at the `` open via the + existing `el.prepend()` head handler. This injection is per-request — origin + HTML in cache is unmodified. +4. TS forces `Transfer-Encoding: chunked` and streams the assembled response + to the browser. +5. The auction runs in parallel regardless of HTML cache state — bids land in + `bid_cache` keyed by `request_id`, served via `/ts-bids` when the client + fetches. + +The `bid_cache` (per-request bid results) and Fastly's HTML cache are +**independent systems**. HTML cache hit/miss does not affect auction firing; +auction firing does not affect HTML caching. + +**`Cache-Control` handling:** + +TS preserves the origin's `Cache-Control` header on the response sent to the +browser, with one override: when `__ts_request_id` is injected (any matched +page), TS sets `Cache-Control: private, no-store` on the **browser-facing** +response to prevent intermediate caches or the browser from caching the +per-user assembled HTML. The Fastly edge cache for the **origin** response is +unaffected — TS reads the cached origin HTML and assembles a fresh per-request +response on every hit. + +`Surrogate-Control` and `Fastly-Surrogate-Control` headers from origin are +preserved (they control Fastly's cache, not the browser's). + +**When caching doesn't apply:** + +- **Logged-in users** — origin typically returns `Cache-Control: private`. Falls + back to cache-miss timing (full origin fetch). +- **Personalized SSR** (per-user content, A/B test variants) — same. +- **Dynamic NextJS routes without ISR** — origin sends `Cache-Control: no-store` + or short max-age. Falls back to cache-miss timing. +- **First request after deploy or cache purge** — cold cache, full origin fetch. +- **Long-tail URLs** — low cache hit rate, treat as cache-miss case. + +For typical news / content publisher sites with anonymous visitors on stable +content pages, expect 70–90%+ edge cache hit rate. The cache-hit timing in §5 +is the realistic common case, not the optimistic best case. + --- ## 5. Request-Time Sequence +Sequence applies to all origins (WordPress, Drupal, Rails, NextJS 14/16, static sites). +TS forces chunked encoding on every response, so origin format is invisible from the +browser's perspective. + +### 5.1 Visual Sequence (full content + creative flow) + +```mermaid +sequenceDiagram + autonumber + participant B as Browser + participant E as TS Edge
(Fastly) + participant C as Fastly HTTP Cache + participant O as Publisher Origin
(WP / NextJS / etc) + participant A as Auction
(PBS + APS) + participant S as SSPs
(Kargo / Index / etc) + participant G as GAM
(securepubads) + + Note over B,G: t=0ms — Navigation start + + B->>E: GET ts.publisher.com/article + + Note over E: t=1ms — URL → slots match
Mint request_id (UUID)
Check consent + + par Auction kicks off server-side + E->>A: POST bid requests
(PBS + APS in parallel) + A->>S: Fan out to all SSPs + S-->>A: Bids return + A-->>E: Aggregated bid responses
(t=502ms) + Note over E: Cache bids in bid_cache
(keyed by request_id, 30s TTL) + E->>S: Fire nurl (fire-and-forget)
for winning bids + and Origin HTML lookup + E->>C: Lookup origin HTML by URL + alt Cache HIT (typical for content pages) + C-->>E: Cached HTML (~5ms) + else Cache MISS (cold / dynamic / logged-in) + C->>O: GET origin HTML + O-->>C: HTML response (~150ms) + C-->>E: HTML response + end + end + + Note over E: Force Transfer-Encoding: chunked
Inject __ts_ad_slots + __ts_request_id
at open
Set Cache-Control: private, no-store + + E-->>B: Stream HTML chunks (no auction wait) + + Note over B: TTFB: ~10ms (hit) / ~155ms (miss)
Browser parses
CSS, fonts, tsjs download
(also from Fastly + browser cache) + + Note over B: flushes immediately
Body parsing begins
🎨 FCP: ~80ms (hit) / ~250ms (miss) + + Note over B: tsjs bundle executes
t=130ms (hit) / t=300ms (miss)
__tsAdInit() defines GPT slots
(no GAM call yet) + + B->>E: GET /ts-bids?rid= + + alt Auction already complete (typical on cache-hit pages) + Note over E: bid_cache hit — return immediately + E-->>B: Bid targeting JSON
(hb_pb, hb_bidder, hb_adid, burl) + else Auction still running + Note over E: Long-poll — block until
auction completes or A_deadline + A-->>E: Bids arrive + E-->>B: Bid targeting JSON
(or {} on timeout) + end + + Note over B: Bids received (~30ms RTT)
setTargeting(hb_*) per slot
Register slotRenderEnded listener
googletag.pubads().refresh() fires + + B->>G: GET /gampad/ads
with hb_* key-values + + Note over G: GAM matches hb_pb against
Prebid line items, selects winner + + G-->>B: Ad markup
(iframe HTML or creative URL) + + Note over B: Creative iframe loads in slot
Fetches sub-resources
(images, scripts, viewability pixels) + + Note over B: 🎯 Creative paints
slotRenderEnded event fires
__tsAdInit checks hb_adid match + + alt Our Prebid bid won the GAM line item match + B->>S: Fire burl (navigator.sendBeacon)
SSP confirms billable impression + else Direct deal / backfill won (hb_adid mismatch or empty) + Note over B: No burl fired — our bid lost
(correct behavior — different creative rendered) + end + + Note over B: window.load fires
(page fully loaded) + + Note over B,G: ✅ AD VISIBLE
Cache hit: ~900ms total
Cache miss: ~1,050ms total
FCP: ~80ms (hit) / ~250ms (miss)

vs client-side today: ~3,250ms ad-visible / FCP ~500ms+ +``` + +### 5.2 Cache-Hit Sequence (typical for content publisher pages) + +This is the common case for anonymous visitors on cacheable content pages. + ``` t=0ms GET ts.publisher.com/article arrives at Fastly edge t=1ms URL matched against creative-opportunities.toml Slots matched: [atf_sidebar_ad, below-content-ad, section_ad] Consent check: TCF consent present → auction proceeds + Request ID minted: 550e8400-e29b-41d4-a716-446655440000 -t=2ms AuctionOrchestrator.run_auction() called +t=2ms AuctionOrchestrator.run_auction() dispatched (parallel) PBS + APS dispatched in parallel via send_async() Edge→PBS RTT: ~20–30ms + Fastly cache lookup dispatched in parallel + __ts_ad_slots + __ts_request_id ".to_string() + r#""# + .to_string() ), - ad_bids_script: None, }; let mut processor = create_html_processor(config); let output = processor .process_chunk(b"T", true) .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); - assert!(html.contains("window.__ts_ad_slots"), "should inject ad slots"); + assert!(html.contains("window.__ts_ad_slots"), "should inject ad slots at head-open"); + assert!(html.contains("window.__ts_request_id"), "should inject request_id at head-open"); } #[test] - fn injects_bids_before_end_of_head() { - let bids_script = ""; + fn does_not_hold_end_of_head() { + // Verify: no bid data appears before — that hold was rejected by spec §4.3 let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: None, - ad_bids_script: Some(bids_script.to_string()), }; let mut processor = create_html_processor(config); let output = processor .process_chunk(b"T", true) .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); - assert!(html.contains("window.__ts_bids"), "should inject bids"); - let bids_pos = html.find("window.__ts_bids").expect("should find bids"); - let end_head_pos = html.find("").expect("should find "); - assert!(bids_pos < end_head_pos, "bids script should appear before "); + assert!(!html.contains("__ts_bids"), "must not inject bids into head"); } ``` Run: `cargo test -p trusted-server-core html_processor` - Expected: compile error (no `ad_slots_script`/`ad_bids_script` fields, no `empty_for_tests()`) + Expected: compile error (no `ad_slots_script` field, no `empty_for_tests()`) - [ ] **Step 2: Add `empty_for_tests()` to `IntegrationRegistry`** @@ -888,7 +886,6 @@ Adding the two new fields to `HtmlProcessorConfig` and the injection logic is in #[cfg(test)] impl IntegrationRegistry { pub fn empty_for_tests() -> Self { - // Minimal registry with no integrations for unit testing html_processor Self { inner: Arc::new(RegistryInner { proxies: Default::default(), @@ -905,7 +902,9 @@ Adding the two new fields to `HtmlProcessorConfig` and the injection logic is in (Adjust field names to match the actual `RegistryInner` struct.) -- [ ] **Step 3: Add fields to `HtmlProcessorConfig`** +- [ ] **Step 3: Add single field to `HtmlProcessorConfig`** + + Replace any existing `ad_slots_script`/`ad_bids_script` fields with: ```rust pub struct HtmlProcessorConfig { @@ -913,56 +912,47 @@ Adding the two new fields to `HtmlProcessorConfig` and the injection logic is in pub request_host: String, pub request_scheme: String, pub integrations: IntegrationRegistry, - /// Pre-computed `` for matched slots. - /// Injected at open, before integration head inserts. `None` when no slots matched. + /// Pre-computed ``. + /// Injected at `` open, before integration head inserts. `None` when no slots matched. pub ad_slots_script: Option, - /// Pre-computed `` for winning bids. - /// Injected immediately before via on_end_tag(). `None` when auction not run. - pub ad_bids_script: Option, } ``` - Update `from_settings` to initialize `ad_slots_script: None, ad_bids_script: None`. + Update `from_settings` (or wherever `HtmlProcessorConfig` is constructed) to initialize `ad_slots_script: None`. -- [ ] **Step 4: Inject `__ts_ad_slots` at head-open AND register `on_end_tag` for `__ts_bids`** +- [ ] **Step 4: Inject `ad_slots_script` at head-open** - In `create_html_processor`, within the EXISTING single `element!("head", ...)` handler, make two changes: - 1. Prepend the ad slots script BEFORE the existing integration inserts: + In `create_html_processor`, within the EXISTING `element!("head", ...)` handler, build the full snippet string with `ad_slots_script` first (so it appears first in output — lol_html `prepend` inserts before children, with **last-prepend-wins** ordering, so we call `prepend` exactly once with the full combined string): - ```rust - // NEW: inject __ts_ad_slots first - if let Some(ref slots_script) = ad_slots_script { - snippet.push_str(slots_script); - } - // ... existing: for insert in integrations.head_inserts(&ctx) { ... } - ``` + ```rust + let ad_slots_script = config.ad_slots_script.clone(); + // ... existing captures ... - 2. After `el.prepend(...)`, register the end-tag handler for `__ts_bids`: - ```rust - // Register on_end_tag handler for __ts_bids injection before - if let Some(bids_script) = ad_bids_script.clone() { - el.on_end_tag(move |end_tag| { - end_tag.before(&bids_script, ContentType::Html); - Ok(()) - })?; - } - ``` + element!("head", |el| { + let mut snippet = String::new(); - Both changes live inside the same `element!("head", ...)` closure — no second handler needed. + // ad_slots_script first so __ts_ad_slots + __ts_request_id appear before + // integration inserts. DO NOT call prepend multiple times — lol_html stacks + // prepend calls in reverse order, so a single prepend with the full string + // guarantees correct ordering. + if let Some(ref slots_script) = ad_slots_script { + snippet.push_str(slots_script); + } - Capture `ad_slots_script` and `ad_bids_script` into the closure the same way as `injected_tsjs`: + // ... existing: for insert in integrations.head_inserts(&ctx) { snippet.push_str(...) } - ```rust - let ad_slots_script = config.ad_slots_script.clone(); - let ad_bids_script = config.ad_bids_script.clone(); + if !snippet.is_empty() { + el.prepend(&snippet, ContentType::Html); + } + // DO NOT register on_end_tag — flushes immediately per spec §4.3 + Ok(()) + }) ``` - > **lol_html `on_end_tag` API note:** `Element::on_end_tag(handler)` is available in lol_html ≥2.0. The handler receives `&mut EndTag` and must return `Result<(), Box>`. Use `ContentType::Html` so the injected `", escaped) + let slots_json_str = serde_json::to_string(&slots_json) + .expect("should serialize ad slots"); + let escaped_slots = html_escape_for_script(&slots_json_str); + // request_id is a UUID (hex + hyphens only) — safe to embed without escaping. + format!( + r#""# + ) } - pub(crate) fn build_ad_bids_script( + /// Build the `BidMap` stored in `bid_cache` and returned by `/ts-bids`. + /// + /// Keyed by slot ID. Values contain `hb_pb`, `hb_bidder`, `hb_adid`, `burl`. + pub(crate) fn build_bid_map( winning_bids: &std::collections::HashMap, price_granularity: crate::price_bucket::PriceGranularity, - ) -> String { - let bids_map: serde_json::Map = winning_bids + ) -> crate::bid_cache::BidMap { + winning_bids .iter() .filter_map(|(slot_id, bid)| { let cpm = bid.price?; - let entry = serde_json::json!({ - "hb_pb": price_bucket(cpm, price_granularity), - "hb_bidder": bid.bidder, - "hb_adid": bid.ad_id.as_deref().unwrap_or(""), - "burl": bid.burl, - }); - Some((slot_id.clone(), entry)) + let entry: std::collections::HashMap = [ + ("hb_pb".to_string(), serde_json::Value::String(price_bucket(cpm, price_granularity))), + ("hb_bidder".to_string(), serde_json::Value::String(bid.bidder.clone())), + ("hb_adid".to_string(), serde_json::Value::String( + bid.ad_id.as_deref().unwrap_or("").to_string() + )), + ("burl".to_string(), bid.burl.as_deref() + .map(serde_json::Value::from) + .unwrap_or(serde_json::Value::Null)), + ].into_iter().collect(); + Some((slot_id.clone(), entry.into_iter() + .map(|(k, v)| (k, v)) + .collect::>() + .into())) }) - .collect(); - let json = serde_json::to_string(&serde_json::Value::Object(bids_map)) - .expect("should serialize bids"); - let escaped = html_escape_for_script(&json); - format!("", escaped) + .collect() } /// HTML-escape a JSON string for safe inline `" .to_string(), - // __tsAdInit definition — reads window.__ts_ad_slots / __ts_bids at call time. + // __tsAdInit: fetches /ts-bids for bid targeting, then drives GPT. + // window.__ts_ad_slots and window.__ts_request_id are injected at head-open by TS. + // bidsPromise resolves concurrently with page rendering — never blocks FCP. concat!( "" @@ -1394,20 +1825,20 @@ The `HtmlProcessorConfig` fields now exist (Task 7). This task wires the auction ```bash git add crates/trusted-server-core/src/integrations/gpt.rs - git commit -m "Emit __tsAdInit function definition from GPT head injector" + git commit -m "Emit __tsAdInit with /ts-bids fetch pattern from GPT head injector" ``` --- -## Task 10: `gpt/index.ts` — TypeScript `__tsAdInit` +## Task 12: `gpt/index.ts` — TypeScript `__tsAdInit` with `/ts-bids` fetch **Files:** - Modify: `crates/js/lib/src/integrations/gpt/index.ts` -The TypeScript version is the authoritative implementation; it must mirror the Rust inline string from Task 9 exactly. +The TypeScript version mirrors the Rust inline string from Task 11. It uses the `bidsPromise` pattern — fetching `/ts-bids` concurrently with GPT slot definition. -- [ ] **Step 1: Write a failing test** +- [ ] **Step 1: Write failing tests** In `crates/js/lib/src/integrations/gpt/index.test.ts`: @@ -1417,20 +1848,21 @@ The TypeScript version is the authoritative implementation; it must mirror the R describe('installTsAdInit', () => { beforeEach(() => { delete (window as any).__ts_ad_slots - delete (window as any).__ts_bids + delete (window as any).__ts_request_id delete (window as any).__tsAdInit }) - it('defines googletag slots from __ts_ad_slots and calls refresh', () => { + it('fetches /ts-bids with request_id and applies bid targeting before refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('atf'), + getTargeting: vi.fn().mockReturnValue([]), } const mockPubads = { enableSingleRequest: vi.fn(), addEventListener: vi.fn(), refresh: vi.fn(), - getTargeting: vi.fn().mockReturnValue([]), } ;(window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, @@ -1447,45 +1879,131 @@ The TypeScript version is the authoritative implementation; it must mirror the R targeting: { pos: 'atf' }, }, ] - ;(window as any).__ts_bids = { - atf: { - hb_pb: '1.00', - hb_bidder: 'kargo', - hb_adid: 'abc', - burl: 'https://ssp/bill', - }, - } + ;(window as any).__ts_request_id = 'test-rid-123' + + const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: async () => ({ + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + burl: 'https://ssp/bill', + }, + }), + } as Response) - // Must import installTsAdInit from the module - const { installTsAdInit } = require('./index') + const { installTsAdInit } = await import('./index') installTsAdInit() - ;(window as any).__tsAdInit() + await (window as any).__tsAdInit() - expect((window as any).googletag.defineSlot).toHaveBeenCalledWith( - '/123/atf', - [[300, 250]], - 'atf' + expect(fetchSpy).toHaveBeenCalledWith( + expect.stringContaining('/ts-bids?rid=test-rid-123'), + expect.objectContaining({ credentials: 'omit' }) ) expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00') expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'kargo') expect(mockPubads.refresh).toHaveBeenCalled() + + fetchSpy.mockRestore() + }) + + it('calls refresh with empty bids when fetch fails', async () => { + const mockPubads = { + enableSingleRequest: vi.fn(), + addEventListener: vi.fn(), + refresh: vi.fn(), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue({ + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + }), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [] + ;(window as any).__ts_request_id = 'rid-fail' + + vi.spyOn(global, 'fetch').mockRejectedValue(new Error('network error')) + + const { installTsAdInit } = await import('./index') + installTsAdInit() + await (window as any).__tsAdInit() + + expect(mockPubads.refresh).toHaveBeenCalled() }) - it('fires burl via sendBeacon on slotRenderEnded when our bid won', () => { + it('fires burl via sendBeacon on slotRenderEnded when our bid won', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) - // ... setup and trigger slotRenderEnded event - // Verify: navigator.sendBeacon called with burl + let capturedListener: ((e: any) => void) | undefined + + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('atf'), + getTargeting: vi.fn().mockReturnValue(['abc']), + } + const mockPubads = { + enableSingleRequest: vi.fn(), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: any) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn + }), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [ + { + id: 'atf', + gam_unit_path: '/123/atf', + div_id: 'atf', + formats: [[300, 250]], + targeting: {}, + }, + ] + ;(window as any).__ts_request_id = 'rid-burl-test' + + vi.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: async () => ({ + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + burl: 'https://ssp/bill', + }, + }), + } as Response) + + const { installTsAdInit } = await import('./index') + installTsAdInit() + await (window as any).__tsAdInit() + + // Trigger slotRenderEnded — slot has our winning hb_adid + expect(capturedListener).toBeDefined() + capturedListener!({ + isEmpty: false, + slot: mockSlot, + }) + + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill') beaconSpy.mockRestore() }) }) ``` Run: `cd crates/js/lib && npx vitest run` - Expected: FAIL — `installTsAdInit` not exported + Expected: FAIL — `installTsAdInit` not exported or fetches wrong endpoint - [ ] **Step 2: Add `installTsAdInit` to `index.ts`** - Add to `crates/js/lib/src/integrations/gpt/index.ts` (bottom of file): + Add to `crates/js/lib/src/integrations/gpt/index.ts`: ```typescript interface TsAdSlot { @@ -1505,60 +2023,87 @@ The TypeScript version is the authoritative implementation; it must mirror the R type TsWindow = Window & { __ts_ad_slots?: TsAdSlot[] - __ts_bids?: Record + __ts_request_id?: string __tsAdInit?: () => void } /** - * Install `window.__tsAdInit` — reads `window.__ts_ad_slots` and `window.__ts_bids` - * (injected by the edge into ), defines GPT slots, applies pre-won bid targeting, - * registers a `slotRenderEnded` listener to fire `burl` via `sendBeacon`, then calls - * `refresh()`. + * Install `window.__tsAdInit`. + * + * Reads `window.__ts_ad_slots` and `window.__ts_request_id` (both injected by + * the edge at `` open). Fetches bid results from `/ts-bids?rid=` + * concurrently with GPT slot definition. Applies targeting and calls `refresh()` + * after the fetch resolves. Registers `slotRenderEnded` to fire `burl` via + * `sendBeacon` when our specific Prebid bid wins the GAM line item match. */ export function installTsAdInit(): void { const w = window as TsWindow w.__tsAdInit = function () { const slots = w.__ts_ad_slots ?? [] - const bids = w.__ts_bids ?? {} + const rid = w.__ts_request_id + + const bidsPromise: Promise> = rid + ? fetch(`/ts-bids?rid=${encodeURIComponent(rid)}`, { + credentials: 'omit', + }) + .then((r) => (r.ok ? r.json() : {})) + .catch(() => ({})) + : Promise.resolve({}) + const g = (window as GptWindow).googletag if (!g) return + g.cmd.push(() => { - slots.forEach((slot) => { - const gptSlot = g.defineSlot?.( - slot.gam_unit_path, - slot.formats, - slot.div_id - ) - if (!gptSlot) return - gptSlot.addService(g.pubads()) - Object.entries(slot.targeting ?? {}).forEach(([k, v]) => - gptSlot.setTargeting(k, v) - ) - const bid = bids[slot.id] ?? {} - ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { - if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + const gptSlots = slots + .map((slot) => { + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats, + slot.div_id + ) + if (!gptSlot) return null + gptSlot.addService(g.pubads()) + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => + gptSlot.setTargeting(k, v) + ) + return { id: slot.id, gptSlot } }) - }) + .filter(Boolean) as Array<{ + id: string + gptSlot: NonNullable> + }> + g.pubads().enableSingleRequest() g.enableServices() - g.pubads().addEventListener?.('slotRenderEnded', (event: any) => { - const slotId: string = event.slot?.getSlotElementId?.() ?? '' - const bid = bids[slotId] ?? {} - if ( - !event.isEmpty && - bid.burl && - event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid - ) { - navigator.sendBeacon(bid.burl) - } + + bidsPromise.then((bids) => { + gptSlots.forEach(({ id, gptSlot }) => { + const bid = bids[id] ?? {} + ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + }) + }) + + g.pubads().addEventListener?.('slotRenderEnded', (event: any) => { + const slotId: string = event.slot?.getSlotElementId?.() ?? '' + const bid = bids[slotId] ?? {} + if ( + !event.isEmpty && + bid.burl && + event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + ) { + navigator.sendBeacon(bid.burl) + } + }) + + g.pubads().refresh() }) - g.pubads().refresh() }) } } ``` - Call `installTsAdInit()` from the integration's initialization path so it's set up when the bundle loads. + Call `installTsAdInit()` from the integration's initialization path. - [ ] **Step 3: Run JS tests** @@ -1574,12 +2119,12 @@ The TypeScript version is the authoritative implementation; it must mirror the R ```bash git add crates/js/lib/src/integrations/gpt/ - git commit -m "Add __tsAdInit and slotRenderEnded burl firing to GPT integration" + git commit -m "Add installTsAdInit with /ts-bids fetch pattern and slotRenderEnded burl firing" ``` --- -## Task 11: `nurl` fire-and-forget +## Task 13: `nurl` fire-and-forget **Files:** @@ -1610,9 +2155,9 @@ The TypeScript version is the authoritative implementation; it must mirror the R fn default_fire_nurl_at_edge() -> bool { true } ``` -- [ ] **Step 3: Fire nurls in publisher.rs after auction** +- [ ] **Step 3: Fire nurls in publisher.rs after bid_cache.put()** - After `auction_result` is obtained, add: + After the `bid_cache.put(...)` call (Task 9 Step 3), add: ```rust if let Some(ref result) = auction_result { @@ -1620,7 +2165,7 @@ The TypeScript version is the authoritative implementation; it must mirror the R } ``` - Add helper (no `.await` — fire-and-forget): + Add helper: ```rust fn fire_winning_nurls( @@ -1671,13 +2216,13 @@ The TypeScript version is the authoritative implementation; it must mirror the R --- -## Task 12: End-to-end integration tests +## Task 14: End-to-end integration tests **Files:** - Modify: `crates/trusted-server-core/src/publisher.rs` (test module) -Tests use `pub(crate)` helpers from Task 8 directly. +Tests use `pub(crate)` helpers from Task 9 directly. - [ ] **Step 1: Write tests** @@ -1686,7 +2231,7 @@ Tests use `pub(crate)` helpers from Task 8 directly. ```rust #[cfg(test)] mod creative_opportunities_tests { - use super::{build_ad_slots_script, build_ad_bids_script, html_escape_for_script}; + use super::{build_head_globals_script, build_bid_map, html_escape_for_script}; use crate::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunitySlot, CreativeOpportunityFormat, CreativeOpportunitiesFile, match_slots, @@ -1719,20 +2264,32 @@ Tests use `pub(crate)` helpers from Task 8 directly. } #[test] - fn ad_slots_script_is_safe_and_parseable() { + fn head_globals_script_contains_ad_slots_and_request_id() { let slots = vec![make_slot()]; let config = make_config(); - let script = build_ad_slots_script(&slots, &config); - assert!(script.contains("window.__ts_ad_slots=JSON.parse"), "should use JSON.parse"); + let rid = "550e8400-e29b-41d4-a716-446655440000"; + let script = build_head_globals_script(&slots, rid, &config); + assert!(script.contains("window.__ts_ad_slots=JSON.parse"), "should use JSON.parse for slots"); assert!(script.contains("atf_sidebar_ad"), "should include slot id"); - // Verify no raw < or > that could break HTML parser - let inner = script.trim_start_matches(""); + assert!(script.contains(&format!("window.__ts_request_id=\"{rid}\"")), "should include request_id"); + assert!(!script.contains("__ts_bids"), "must NOT contain bids — bids come from /ts-bids"); + } + + #[test] + fn head_globals_script_is_xss_safe() { + let slots = vec![make_slot()]; + let config = make_config(); + let script = build_head_globals_script(&slots, "safe-rid", &config); + // Strip outer "); assert!(!inner.contains('<'), "no unescaped < in script content"); assert!(!inner.contains('>'), "no unescaped > in script content"); } #[test] - fn ad_bids_script_uses_price_bucket_and_ad_id() { + fn bid_map_uses_price_bucket_and_ad_id() { let mut winning_bids = HashMap::new(); winning_bids.insert("atf_sidebar_ad".to_string(), Bid { slot_id: "atf_sidebar_ad".to_string(), @@ -1747,11 +2304,23 @@ Tests use `pub(crate)` helpers from Task 8 directly. ad_id: Some("prebid-uuid-abc123".to_string()), metadata: HashMap::new(), }); - let script = build_ad_bids_script(&winning_bids, PriceGranularity::Dense); - assert!(script.contains("\"hb_pb\":\"2.53\""), "should bucket 2.53 as 2.53 (dense)"); - assert!(script.contains("\"hb_bidder\":\"kargo\""), "should include bidder"); - assert!(script.contains("\"hb_adid\":\"prebid-uuid-abc123\""), "should use ad_id not creative markup"); - assert!(script.contains("burl"), "should include burl for billing"); + let bid_map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let slot_bids = bid_map.get("atf_sidebar_ad").expect("should have slot bids"); + assert_eq!( + slot_bids.get("hb_pb").and_then(|v| v.as_str()), + Some("2.53"), + "should bucket 2.53 as 2.53 (dense)" + ); + assert_eq!( + slot_bids.get("hb_bidder").and_then(|v| v.as_str()), + Some("kargo"), + "should include bidder" + ); + assert_eq!( + slot_bids.get("hb_adid").and_then(|v| v.as_str()), + Some("prebid-uuid-abc123"), + "should use ad_id not creative markup" + ); } #[test] @@ -1795,7 +2364,7 @@ Tests use `pub(crate)` helpers from Task 8 directly. ```bash git add crates/trusted-server-core/src/publisher.rs - git commit -m "Add integration tests for creative opportunities pipeline (slots, bids, XSS)" + git commit -m "Add integration tests for creative opportunities pipeline (head globals, bid map, XSS)" ``` --- @@ -1804,19 +2373,27 @@ Tests use `pub(crate)` helpers from Task 8 directly. Run `fastly compute serve` and verify: -- [ ] **No match:** Request `/about` — no `__ts_ad_slots` or `__ts_bids` in response HTML, no `Cache-Control: private, no-store` -- [ ] **Match:** Request `/2024/01/article` — both globals present in ``, `Cache-Control: private, no-store` set -- [ ] **Empty file kill-switch:** Empty `creative-opportunities.toml` → no globals injected on any URL -- [ ] **Auction timeout:** Set `auction_timeout_ms = 1` → `__ts_bids` injects as `{}`, no slot entries -- [ ] **XSS check:** Add `targeting = { zone = " -``` - -> **Security:** All string values are JSON-serialized via `serde_json` and HTML-escaped -> before insertion into the `, ContentType::Html)`. + +> **Security:** All string values are JSON-serialized via `serde_json` and HTML-escaped +> before insertion into the `"# - .to_string() + r#""#.to_string() ), + ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), }; let mut processor = create_html_processor(config); let output = processor - .process_chunk(b"T", true) + .process_chunk(b"Tcontent", true) .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert!(html.contains("window.__ts_ad_slots"), "should inject ad slots at head-open"); - assert!(html.contains("window.__ts_request_id"), "should inject request_id at head-open"); + assert!(!html.contains("__ts_request_id"), "must NOT inject request_id — body-injection arch has no request_id"); } #[test] - fn does_not_hold_end_of_head() { - // Verify: no bid data appears before — that hold was rejected by spec §4.3 + fn injects_ts_bids_before_body_close() { + let bids_script = r#""#; + let state = std::sync::Arc::new(std::sync::RwLock::new( + Some(bids_script.to_string()) + )); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: None, + ad_bids_state: state, }; let mut processor = create_html_processor(config); let output = processor - .process_chunk(b"T", true) + .process_chunk(b"content", true) .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); - assert!(!html.contains("__ts_bids"), "must not inject bids into head"); + assert!(html.contains("window.__ts_bids"), "should inject bids before "); + let bids_pos = html.find("window.__ts_bids").expect("bids should be in output"); + let body_close_pos = html.find("").expect(" should be in output"); + assert!(bids_pos < body_close_pos, "bids must appear before "); + } + + #[test] + fn injects_empty_ts_bids_when_state_is_none() { + let state = std::sync::Arc::new(std::sync::RwLock::new(None)); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!(html.contains("__ts_bids=JSON.parse(\"{}\""), "should inject empty bids on None state"); } ``` Run: `cargo test -p trusted-server-core html_processor` - Expected: compile error (no `ad_slots_script` field, no `empty_for_tests()`) + Expected: compile error (no `ad_bids_state` field yet) - [ ] **Step 2: Add `empty_for_tests()` to `IntegrationRegistry`** @@ -902,9 +930,7 @@ The `hb_pb` value in bid responses is a discretized bucket string from Prebid's (Adjust field names to match the actual `RegistryInner` struct.) -- [ ] **Step 3: Add single field to `HtmlProcessorConfig`** - - Replace any existing `ad_slots_script`/`ad_bids_script` fields with: +- [ ] **Step 3: Update `HtmlProcessorConfig`** ```rust pub struct HtmlProcessorConfig { @@ -912,362 +938,104 @@ The `hb_pb` value in bid responses is a discretized bucket string from Prebid's pub request_host: String, pub request_scheme: String, pub integrations: IntegrationRegistry, - /// Pre-computed ``. - /// Injected at `` open, before integration head inserts. `None` when no slots matched. + /// Pre-computed ``. + /// Injected at `` open. `None` when no slots matched. pub ad_slots_script: Option, + /// Shared auction result script — written by the auction task before HTML processing + /// begins. Handler reads this in `el.on_end_tag()` on the body element. + /// `None` means no auction ran (consent denied, bot UA, no slot match, etc.); + /// inject empty `__ts_bids = {}` as graceful fallback. + pub ad_bids_state: std::sync::Arc>>, } ``` - Update `from_settings` (or wherever `HtmlProcessorConfig` is constructed) to initialize `ad_slots_script: None`. + Update `from_settings` (or wherever `HtmlProcessorConfig` is constructed) to initialize `ad_bids_state: Arc::new(RwLock::new(None))`. - [ ] **Step 4: Inject `ad_slots_script` at head-open** - In `create_html_processor`, within the EXISTING `element!("head", ...)` handler, build the full snippet string with `ad_slots_script` first (so it appears first in output — lol_html `prepend` inserts before children, with **last-prepend-wins** ordering, so we call `prepend` exactly once with the full combined string): + In `create_html_processor`, within the existing `element!("head", ...)` handler: ```rust let ad_slots_script = config.ad_slots_script.clone(); - // ... existing captures ... + // existing captures... element!("head", |el| { let mut snippet = String::new(); - - // ad_slots_script first so __ts_ad_slots + __ts_request_id appear before - // integration inserts. DO NOT call prepend multiple times — lol_html stacks - // prepend calls in reverse order, so a single prepend with the full string - // guarantees correct ordering. if let Some(ref slots_script) = ad_slots_script { snippet.push_str(slots_script); } - - // ... existing: for insert in integrations.head_inserts(&ctx) { snippet.push_str(...) } - + // existing integration head inserts... if !snippet.is_empty() { el.prepend(&snippet, ContentType::Html); } - // DO NOT register on_end_tag — flushes immediately per spec §4.3 + // DO NOT register on_end_tag — flushes immediately Ok(()) }) ``` -- [ ] **Step 5: Run tests** - - Run: `cargo test -p trusted-server-core html_processor` - Expected: all tests pass (including the new ones; no bids injection test must also pass) - -- [ ] **Step 6: Run full suite** - - Run: `cargo test --workspace` - Expected: clean - -- [ ] **Step 7: Commit** - - ```bash - git add crates/trusted-server-core/src/html_processor.rs \ - crates/trusted-server-core/src/integrations/registry.rs - git commit -m "Add ad_slots_script injection to HtmlProcessorConfig at head-open; no hold" - ``` - ---- - -## Task 8: `bid_cache.rs` — In-process auction result cache - -**Files:** - -- Create: `crates/trusted-server-core/src/bid_cache.rs` -- Modify: `crates/trusted-server-core/src/lib.rs` - -The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL. It is shared across concurrent Fastly request handlers via `std::sync::Mutex`. The `/ts-bids` endpoint (Task 10) uses `wait_for()` to block-poll until results arrive or the deadline fires. - -> **WASM note:** `std::time::Instant` and `std::thread::sleep` are both supported in Viceroy and Fastly Compute. The Mutex is uncontested in practice — requests are handled cooperatively with brief lock windows. - -- [ ] **Step 1: Write failing tests** +- [ ] **Step 5: Inject `__ts_bids` before `` via `el.on_end_tag()`** - Create `crates/trusted-server-core/src/bid_cache.rs` with only the tests: + Add a new handler in `create_html_processor`. The shared state is already populated by the time lol_html reaches `` (Task 9 awaits the auction before starting HTML processing): ```rust - #[cfg(test)] - mod tests { - use super::*; - use std::time::{Duration, Instant}; - - fn make_bids() -> BidMap { - let mut m = std::collections::HashMap::new(); - m.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); - m - } - - #[test] - fn returns_not_found_for_unknown_rid() { - let cache = BidCache::new(Duration::from_secs(30), 100); - let result = cache.try_get("unknown-rid"); - assert!(matches!(result, CacheResult::NotFound), "should return NotFound"); - } - - #[test] - fn returns_pending_before_put() { - let cache = BidCache::new(Duration::from_secs(30), 100); - let deadline = Instant::now() + Duration::from_secs(5); - cache.mark_pending("rid-1", deadline); - let result = cache.try_get("rid-1"); - assert!(matches!(result, CacheResult::Pending), "should be Pending"); - } - - #[test] - fn returns_bids_after_put() { - let cache = BidCache::new(Duration::from_secs(30), 100); - let deadline = Instant::now() + Duration::from_secs(5); - cache.mark_pending("rid-2", deadline); - cache.put("rid-2", make_bids()); - match cache.try_get("rid-2") { - CacheResult::Complete(bids) => { - assert!(bids.contains_key("atf"), "should contain atf bid"); + let ad_bids_state = config.ad_bids_state.clone(); + + element!("body", |el| { + let state = ad_bids_state.clone(); + el.on_end_tag(move |end_tag| { + let script = state.read().expect("should read bid state"); + let bids_script = match &*script { + Some(s) => s.clone(), + None => { + r#""#.to_string() } - other => panic!("expected Complete, got {:?}", other), - } - } - - #[test] - fn returns_not_found_for_expired_entry() { - let cache = BidCache::new(Duration::from_millis(1), 100); - let deadline = Instant::now() + Duration::from_secs(5); - cache.mark_pending("rid-3", deadline); - cache.put("rid-3", make_bids()); - std::thread::sleep(Duration::from_millis(5)); - let result = cache.try_get("rid-3"); - assert!(matches!(result, CacheResult::NotFound), "should expire after TTL"); - } - - #[test] - fn wait_for_returns_bids_immediately_when_complete() { - let cache = BidCache::new(Duration::from_secs(30), 100); - let deadline = Instant::now() + Duration::from_secs(5); - cache.mark_pending("rid-4", deadline); - cache.put("rid-4", make_bids()); - let result = cache.wait_for("rid-4", deadline); - assert!(matches!(result, WaitResult::Bids(_)), "should return bids immediately"); - } - - #[test] - fn wait_for_returns_not_found_for_unknown_rid() { - let cache = BidCache::new(Duration::from_secs(30), 100); - let deadline = Instant::now() + Duration::from_millis(50); - let result = cache.wait_for("never-registered", deadline); - assert!(matches!(result, WaitResult::NotFound), "should return NotFound"); - } - } - ``` - - Run: `cargo test -p trusted-server-core bid_cache` - Expected: compile error (module not exported yet) - -- [ ] **Step 2: Implement bid_cache.rs** - - ```rust - //! In-process auction result cache keyed by request ID. - //! - //! Shared across concurrent Fastly request handlers via a global `Mutex`. - //! Entries expire after a configurable TTL (30 seconds by default). - - use std::collections::HashMap; - use std::sync::Mutex; - use std::time::{Duration, Instant}; - - pub type BidMap = HashMap; - - #[derive(Debug)] - enum EntryState { - Pending { auction_deadline: Instant }, - Complete { bids: BidMap }, - } - - struct CacheEntry { - state: EntryState, - inserted_at: Instant, - } - - struct BidCacheInner { - entries: HashMap, - insertion_order: std::collections::VecDeque, - capacity: usize, - ttl: Duration, - } - - impl BidCacheInner { - fn evict_expired(&mut self) { - let now = Instant::now(); - self.insertion_order.retain(|rid| { - self.entries.get(rid) - .map(|e| now.duration_since(e.inserted_at) < self.ttl) - .unwrap_or(false) - }); - self.entries.retain(|_, e| now.duration_since(e.inserted_at) < self.ttl); - } - - fn evict_oldest_if_full(&mut self) { - while self.entries.len() >= self.capacity { - if let Some(oldest) = self.insertion_order.pop_front() { - self.entries.remove(&oldest); - } else { - break; - } - } - } - } - - /// Outcome of a non-blocking cache lookup. - #[derive(Debug)] - pub enum CacheResult { - /// Auction complete; bids are ready. - Complete(BidMap), - /// Auction registered but not yet complete. - Pending, - /// Request ID never registered, or TTL expired. - NotFound, - } - - /// Outcome of a blocking `wait_for` call. - #[derive(Debug)] - pub enum WaitResult { - /// Auction completed within the deadline. - Bids(BidMap), - /// Deadline passed; bids not available. - Empty, - /// Request ID never registered (caller should return 404). - NotFound, - } - - /// In-process cache for auction results, shared across request handlers. - pub struct BidCache { - inner: Mutex, - } - - impl BidCache { - /// Create a new `BidCache`. - /// - /// # Arguments - /// - `ttl`: how long to keep entries before expiry - /// - `capacity`: max number of concurrent entries (oldest evicted when full) - pub fn new(ttl: Duration, capacity: usize) -> Self { - Self { - inner: Mutex::new(BidCacheInner { - entries: HashMap::new(), - insertion_order: std::collections::VecDeque::new(), - capacity, - ttl, - }), - } - } - - /// Register a request as in-flight. Call at auction start, before `run_auction`. - pub fn mark_pending(&self, request_id: &str, auction_deadline: Instant) { - let mut inner = self.inner.lock().expect("should lock bid_cache"); - inner.evict_expired(); - inner.evict_oldest_if_full(); - inner.entries.insert(request_id.to_string(), CacheEntry { - state: EntryState::Pending { auction_deadline }, - inserted_at: Instant::now(), - }); - inner.insertion_order.push_back(request_id.to_string()); - } - - /// Store completed auction results. Transitions entry from Pending → Complete. - pub fn put(&self, request_id: &str, bids: BidMap) { - let mut inner = self.inner.lock().expect("should lock bid_cache"); - if let Some(entry) = inner.entries.get_mut(request_id) { - entry.state = EntryState::Complete { bids }; - } - } - - /// Non-blocking lookup. Returns current state without sleeping. - pub fn try_get(&self, request_id: &str) -> CacheResult { - let inner = self.inner.lock().expect("should lock bid_cache"); - let now = Instant::now(); - match inner.entries.get(request_id) { - None => CacheResult::NotFound, - Some(entry) if now.duration_since(entry.inserted_at) >= inner.ttl => { - CacheResult::NotFound - } - Some(entry) => match &entry.state { - EntryState::Pending { .. } => CacheResult::Pending, - EntryState::Complete { bids } => CacheResult::Complete(bids.clone()), - }, - } - } - - /// Return the stored auction deadline for a pending entry (the `T₀ + auction_timeout_ms` - /// value minted when the page request arrived). Used by `/ts-bids` to enforce the correct - /// deadline rather than minting a fresh `Instant::now() + timeout`. - /// - /// Returns `None` if the entry is unknown, expired, or already complete. - pub fn get_auction_deadline(&self, request_id: &str) -> Option { - let inner = self.inner.lock().expect("should lock bid_cache"); - let now = Instant::now(); - inner.entries.get(request_id).and_then(|entry| { - if now.duration_since(entry.inserted_at) >= inner.ttl { - return None; - } - match entry.state { - EntryState::Pending { auction_deadline } => Some(auction_deadline), - EntryState::Complete { .. } => None, - } - }) - } - - /// Block until bids are available for `request_id` or `deadline` passes. - /// - /// Polls every 50ms. Returns `NotFound` immediately if `request_id` was never registered. - /// Returns `Empty` if deadline fires before auction completes. - pub fn wait_for(&self, request_id: &str, deadline: Instant) -> WaitResult { - loop { - match self.try_get(request_id) { - CacheResult::Complete(bids) => return WaitResult::Bids(bids), - CacheResult::NotFound => return WaitResult::NotFound, - CacheResult::Pending => { - if Instant::now() >= deadline { - return WaitResult::Empty; - } - std::thread::sleep(Duration::from_millis(50)); - } - } - } - } - } + }; + end_tag.before(&bids_script, ContentType::Html); + Ok(()) + })?; + Ok(()) + }) ``` -- [ ] **Step 3: Export from lib.rs** +- [ ] **Step 6: Run tests** - ```rust - pub mod bid_cache; - ``` + Run: `cargo test -p trusted-server-core html_processor` + Expected: all tests pass -- [ ] **Step 4: Run tests** +- [ ] **Step 7: Run full suite** - Run: `cargo test -p trusted-server-core bid_cache` - Expected: all tests pass + Run: `cargo test --workspace` + Expected: clean -- [ ] **Step 5: Commit** +- [ ] **Step 8: Commit** ```bash - git add crates/trusted-server-core/src/bid_cache.rs \ - crates/trusted-server-core/src/lib.rs - git commit -m "Add BidCache with 30s TTL, pending/complete states, and blocking wait_for" + git add crates/trusted-server-core/src/html_processor.rs \ + crates/trusted-server-core/src/integrations/registry.rs + git commit -m "Inject __ts_ad_slots at head-open and __ts_bids before via shared auction state" ``` --- -## Task 9: `handle_publisher_request` async restructuring +## Task 8: `handle_publisher_request` async restructuring **Files:** - Modify: `crates/trusted-server-core/src/publisher.rs` - Modify: `crates/trusted-server-adapter-fastly/src/main.rs` -> **Key constraint from spec §4.3:** Page rendering is never held for the auction. The auction and origin fetch run concurrently via Fastly's `send_async()` model — origin is dispatched first (non-blocking), then the auction runs its own `send_async` calls, so both overlap on the network. Bid results go to `bid_cache` only — they are NOT injected into the HTML. `Cache-Control: private, no-store` is set whenever slots matched (not just when bids arrived). +> **Key constraint from spec §4.3 and §3:** No `bid_cache`. No `/ts-bids`. No `request_id`. Bids travel inline with the HTML response via body injection. The `Arc>>` is the coordination mechanism within a single request's lifetime — it is written before HTML processing and read by the lol_html `` handler. + +> **Eligibility gating (spec §4.3):** Auctions fire only for real GET requests from non-bot, non-prefetch clients with TCF Purpose 1 consent and at least one matching slot. All other requests proceed with no auction and no `__ts_bids` injection. + +> **Cache-Control (spec §4.7):** Set `Cache-Control: private, max-age=0` (not `no-store`) to preserve BFCache eligibility. Strip `Surrogate-Control` and `Fastly-Surrogate-Control`. - [ ] **Step 1: Update function signature** Change `handle_publisher_request` in `publisher.rs`: + > **Existing context:** The existing `publisher.rs` function body already computes `consent_context`, `ec_id`, `request_info`, `origin_host`, and `backend_name` before the origin fetch. Steps below insert new logic between those existing computations and the origin fetch — they do not replace them. + ```rust pub async fn handle_publisher_request( settings: &Settings, @@ -1275,31 +1043,48 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL services: &RuntimeServices, orchestrator: &crate::auction::orchestrator::AuctionOrchestrator, slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, - bid_cache: &crate::bid_cache::BidCache, mut req: Request, ) -> Result> ``` - Add imports: + Add imports at top of file: ```rust + use std::sync::{Arc, RwLock}; + use fastly::http::header; use crate::auction::orchestrator::AuctionOrchestrator; use crate::auction::types::{AuctionContext, AuctionRequest, PublisherInfo, UserInfo, SiteInfo}; - use crate::bid_cache::{BidCache, BidMap}; use crate::creative_opportunities::{CreativeOpportunitiesFile, match_slots}; use crate::price_bucket::price_bucket; ``` -- [ ] **Step 2: Mint `request_id`, match URL, check consent** + > **`send_async` return type:** `req.send_async()` returns `fastly::handle::PendingRequestHandle` (re-exported as `fastly::PendingRequest` in recent versions). Confirm the exact type from the `fastly` crate version in `Cargo.toml`; `.wait()` is the blocking resolve method on whichever type is returned. - At the top of the function body, before the origin fetch: +- [ ] **Step 2: Apply auction-eligibility gates** - ```rust - // Mint per-request UUID — included in head injection and /ts-bids lookup key. - let request_id = uuid::Uuid::new_v4().to_string(); + At the top of the function body, before origin fetch: + ```rust let request_path = req.get_path().to_string(); - let matched_slots: Vec<_> = if settings.creative_opportunities.is_some() { + let request_method = req.get_method().clone(); + + // Gate 1: Only GET triggers auctions. HEAD skips everything. + let is_get = request_method == fastly::http::Method::GET; + + // Gate 2: Skip prefetch hints (Sec-Purpose: prefetch or Purpose: prefetch). + let is_prefetch = req.get_header_str("sec-purpose") + .map_or(false, |v| v.contains("prefetch")) + || req.get_header_str("purpose") + .map_or(false, |v| v.contains("prefetch")); + + // Gate 3: Skip well-known crawler UAs (protects SSP QPS budget). + let user_agent = req.get_header_str("user-agent").unwrap_or(""); + let is_bot = ["Googlebot", "Bingbot", "AhrefsBot", "SemrushBot", "DotBot"] + .iter() + .any(|bot| user_agent.contains(bot)); + + // Gate 4: Slot match. + let matched_slots: Vec<_> = if settings.creative_opportunities.is_some() && is_get { match_slots(&slots_file.slots, &request_path) .into_iter() .cloned() @@ -1308,11 +1093,17 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL Vec::new() }; + // Gate 5: TCF Purpose 1 consent. let consent_allows_auction = consent_context .tcf .as_ref() .map_or(false, |tcf| tcf.has_purpose_consent(1)); - let should_run_auction = !matched_slots.is_empty() && consent_allows_auction; + + let should_run_auction = is_get + && !is_prefetch + && !is_bot + && !matched_slots.is_empty() + && consent_allows_auction; let auction_timeout_ms = settings .creative_opportunities @@ -1321,33 +1112,24 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL .unwrap_or(settings.auction.timeout_ms); ``` -- [ ] **Step 3: Register pending in bid_cache, fire origin + auction concurrently** +- [ ] **Step 3: Create shared bid state, fire origin + auction concurrently** ```rust - // Mint T₀ auction deadline. Stored in bid_cache so /ts-bids uses the same deadline, - // not a freshly-minted one when the browser's fetch arrives. - let auction_deadline = std::time::Instant::now() - + std::time::Duration::from_millis(u64::from(auction_timeout_ms)); - - // Register request as in-flight so /ts-bids can long-poll for it. - if should_run_auction { - bid_cache.mark_pending(&request_id, auction_deadline); - } + // Shared state: auction task writes the ready-to-inject script; lol_html + // handler reads it. Both within the same request — no cross-request sharing. + let ad_bids_state: Arc>> = Arc::new(RwLock::new(None)); restrict_accept_encoding(&mut req); req.set_header("host", &origin_host); - // Fire origin request immediately — Fastly's send_async dispatches the HTTP request - // to the network without blocking. The origin fetch is in-flight from this point. - // The auction below also uses send_async internally, so both origin SSP requests - // overlap on the network. This is Fastly's concurrency model — no join! needed. + // Fire origin immediately — both origin and auction SSP calls overlap on the network. let pending_origin = req .send_async(&backend_name) .change_context(TrustedServerError::Proxy { message: "Failed to dispatch async origin request".to_string(), })?; - // Run auction (internal send_async calls overlap with origin fetch on the network). + // Run auction. Internal SSP calls use send_async and overlap with origin fetch. let auction_result = if should_run_auction { let co_config = settings.creative_opportunities.as_ref() .expect("should be present when should_run_auction is true"); @@ -1377,17 +1159,20 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL None }; - // Write auction results to bid_cache — /ts-bids will serve them. + // Write auction result to shared state before HTML processing begins. + // The lol_html handler reads this synchronously — it is always populated here. + // `build_bid_map` returns `serde_json::Map`. if should_run_auction { let co_config = settings.creative_opportunities.as_ref() .expect("should be present"); - // Bind empty map to a local to avoid &Default::default() referencing a temporary. - let empty_bids = std::collections::HashMap::new(); + let empty_bids: std::collections::HashMap = + std::collections::HashMap::new(); let winning_bids = auction_result.as_ref() .map(|r| &r.winning_bids) .unwrap_or(&empty_bids); let bid_map = build_bid_map(winning_bids, co_config.price_granularity); - bid_cache.put(&request_id, bid_map); + let bids_script = build_bids_script(&bid_map); + *ad_bids_state.write().expect("should write bid state") = Some(bids_script); } // Await origin response (may already be buffered since we started it before the auction). @@ -1403,10 +1188,9 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL After acquiring `response`: ```rust - // Build head injection script: __ts_ad_slots + __ts_request_id (never bids). let ad_slots_script = if let Some(co_config) = &settings.creative_opportunities { if !matched_slots.is_empty() { - Some(build_head_globals_script(&matched_slots, &request_id, co_config)) + Some(build_ad_slots_script(&matched_slots, co_config)) } else { None } @@ -1414,33 +1198,91 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL None }; - // When slots matched: prevent browser/CDN caching of the per-user assembled HTML. - // Spec §4.4: set regardless of whether bids arrived — the request_id is now in the page. + // Set cache headers when slots matched. private, max-age=0 (not no-store) preserves + // BFCache eligibility — browser back/forward cache restores the already-rendered ad + // without firing a new GAM call, which is the desired behavior. if ad_slots_script.is_some() { - response.set_header(header::CACHE_CONTROL, "private, no-store"); + response.set_header(header::CACHE_CONTROL, "private, max-age=0"); response.remove_header("surrogate-control"); response.remove_header("fastly-surrogate-control"); } - // Spec §4.3/§4.7: Force chunked encoding on every origin response so that - // reaches the browser immediately as chunks arrive — regardless of whether origin - // sent a buffered response (WordPress, Drupal) or a streaming one (NextJS 16). - // Removing Content-Length is required; sending both headers is invalid HTTP/1.1. + // Force chunked encoding so reaches the browser immediately as chunks arrive. + // Sending both Content-Length and Transfer-Encoding is invalid HTTP/1.1. response.remove_header(header::CONTENT_LENGTH); response.set_header("transfer-encoding", "chunked"); ``` -- [ ] **Step 5: Add `pub(crate)` helper functions** +- [ ] **Step 5: Thread shared state into `OwnedProcessResponseParams`** + + Update `OwnedProcessResponseParams`: + + ```rust + pub struct OwnedProcessResponseParams { + // existing fields... + pub(crate) ad_slots_script: Option, + pub(crate) ad_bids_state: Arc>>, + } + ``` + + Pass both through to `create_html_stream_processor` and into `HtmlProcessorConfig`. + +- [ ] **Step 6: Add `pub(crate)` helper functions** + + > **`BidMap` type:** Use `serde_json::Map` directly — no separate module needed. + + Add helpers in this order (each function is used by the one below it, so define leaf functions first): ```rust + /// HTML-escape a JSON string for safe inline `"#) + } + /// Build the `"# - ) - } - - /// Build the `BidMap` stored in `bid_cache` and returned by `/ts-bids`. - /// - /// Keyed by slot ID. Values contain `hb_pb`, `hb_bidder`, `hb_adid`, `burl`. - pub(crate) fn build_bid_map( - winning_bids: &std::collections::HashMap, - price_granularity: crate::price_bucket::PriceGranularity, - ) -> crate::bid_cache::BidMap { - winning_bids - .iter() - .filter_map(|(slot_id, bid)| { - let cpm = bid.price?; - let entry: std::collections::HashMap = [ - ("hb_pb".to_string(), serde_json::Value::String(price_bucket(cpm, price_granularity))), - ("hb_bidder".to_string(), serde_json::Value::String(bid.bidder.clone())), - ("hb_adid".to_string(), serde_json::Value::String( - bid.ad_id.as_deref().unwrap_or("").to_string() - )), - ("burl".to_string(), bid.burl.as_deref() - .map(serde_json::Value::from) - .unwrap_or(serde_json::Value::Null)), - ].into_iter().collect(); - Some((slot_id.clone(), entry.into_iter() - .map(|(k, v)| (k, v)) - .collect::>() - .into())) - }) - .collect() - } - - /// HTML-escape a JSON string for safe inline `"#) } fn build_auction_request( @@ -1535,39 +1332,25 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL } ``` -- [ ] **Step 6: Thread `ad_slots_script` into `OwnedProcessResponseParams`** - - Update `OwnedProcessResponseParams`: - - ```rust - pub struct OwnedProcessResponseParams { - // existing fields... - pub(crate) ad_slots_script: Option, - } - ``` - - Pass `ad_slots_script` through to `create_html_stream_processor` and into `HtmlProcessorConfig`. + > **Type note:** All helper signatures use `serde_json::Map` directly. Do not create a `BidMap` type alias or `bid_types.rs` module. - [ ] **Step 7: Update `main.rs` call site** In `crates/trusted-server-adapter-fastly/src/main.rs`: ```rust - // At startup — load creative-opportunities.toml and initialize bid_cache. + // At startup (top of main() / request handler setup, before the request dispatch loop). + // include_str! embeds the file at compile time — no runtime file I/O. const CREATIVE_OPPORTUNITIES_TOML: &str = include_str!("../../../creative-opportunities.toml"); - let slots_file: creative_opportunities::CreativeOpportunitiesFile = + let slots_file: trusted_server_core::creative_opportunities::CreativeOpportunitiesFile = toml::from_str(CREATIVE_OPPORTUNITIES_TOML) .expect("should parse creative-opportunities.toml"); - - // BidCache: 30s TTL, capacity 1000 entries (each entry is a few KB). - let bid_cache = crate::bid_cache::BidCache::new( - std::time::Duration::from_secs(30), - 1000, - ); ``` + `slots_file` is a local in the startup/handler scope and passed by reference into `handle_publisher_request` on each request — no `Arc` needed since it's immutable and the handler borrows it. + Update the call to `handle_publisher_request`: ```rust @@ -1575,15 +1358,16 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL settings, integration_registry, &publisher_services, - orchestrator, // existing - &slots_file, // new - &bid_cache, // new + orchestrator, // existing + &slots_file, // new req, ).await { // existing match arms unchanged } ``` + There is **no `/ts-bids` route** to add. The body injection is complete within `handle_publisher_request`. + - [ ] **Step 8: Compile check** Run: `cargo check --workspace` @@ -1599,164 +1383,43 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL ```bash git add crates/trusted-server-core/src/publisher.rs \ crates/trusted-server-adapter-fastly/src/main.rs - git commit -m "Convert handle_publisher_request to async; auction writes to bid_cache; inject head globals only" + git commit -m "Convert handle_publisher_request to async; body-inject __ts_bids; eligibility gates; max-age=0" ``` --- -## Task 10: `/ts-bids` endpoint - -**Files:** - -- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` - -The `/ts-bids` endpoint is the client's fetch target for bid results. It long-polls until the auction completes or the deadline fires, then returns JSON. Bid results were already stored in `bid_cache` by Task 9. - -- [ ] **Step 1: Write failing test (integration-style)** - - In `main.rs` test module (or a new `tests/ts_bids.rs`): - - ```rust - #[test] - fn ts_bids_response_structure() { - use crate::bid_cache::{BidCache, WaitResult}; - use std::time::{Duration, Instant}; - - let cache = BidCache::new(Duration::from_secs(30), 100); - let rid = "test-rid-abc"; - let deadline = Instant::now() + Duration::from_secs(5); - cache.mark_pending(rid, deadline); - let mut bids = std::collections::HashMap::new(); - bids.insert("atf".to_string(), serde_json::json!({ - "hb_pb": "1.00", "hb_bidder": "kargo", "hb_adid": "abc", "burl": null, - })); - cache.put(rid, bids); - - match cache.wait_for(rid, deadline) { - WaitResult::Bids(b) => { - assert!(b.contains_key("atf"), "should contain atf slot bids"); - } - other => panic!("expected Bids, got {:?}", other), - } - } - ``` - - Run: `cargo test -p trusted-server-adapter-fastly ts_bids` - Expected: compile error (no handler yet, or pass since it's testing bid_cache directly) - -- [ ] **Step 2: Add `/ts-bids` route handler in `main.rs`** - - In the request routing section, before the publisher fallback, add: - - ```rust - if req.get_path() == "/ts-bids" && req.get_method() == fastly::http::Method::GET { - return handle_ts_bids_request(req, &bid_cache, settings); - } - ``` - - Add the handler function: - - ```rust - fn handle_ts_bids_request( - req: fastly::Request, - bid_cache: &crate::bid_cache::BidCache, - settings: &Settings, - ) -> fastly::Response { - // Parse `rid` query param. - let rid = req.get_query_parameter("rid").map(String::from); - let rid = match rid { - Some(r) if !r.is_empty() => r, - _ => { - return fastly::Response::from_status(fastly::http::StatusCode::BAD_REQUEST) - .with_body_text_plain("missing rid parameter"); - } - }; - - // Use the stored T₀ auction deadline from bid_cache — not a freshly-minted - // Instant::now() + timeout, which would extend the window past the original A_deadline. - // Spec §4.4: "/ts-bids blocks until auction completion or A_deadline" where A_deadline - // = T₀ + auction_timeout_ms (minted at page request receipt, stored in bid_cache entry). - let deadline = bid_cache.get_auction_deadline(&rid) - .unwrap_or_else(|| { - // Fallback: rid is unknown or already complete. wait_for returns immediately. - std::time::Instant::now() - }); - - let result = bid_cache.wait_for(&rid, deadline); - - match result { - crate::bid_cache::WaitResult::Bids(bids) => { - let body = serde_json::to_string(&bids) - .unwrap_or_else(|_| "{}".to_string()); - fastly::Response::from_status(fastly::http::StatusCode::OK) - .with_header(fastly::http::header::CONTENT_TYPE, "application/json") - .with_header(fastly::http::header::CACHE_CONTROL, "private, no-store") - .with_body(body) - } - crate::bid_cache::WaitResult::Empty => { - fastly::Response::from_status(fastly::http::StatusCode::OK) - .with_header(fastly::http::header::CONTENT_TYPE, "application/json") - .with_header(fastly::http::header::CACHE_CONTROL, "private, no-store") - .with_body("{}") - } - crate::bid_cache::WaitResult::NotFound => { - fastly::Response::from_status(fastly::http::StatusCode::NOT_FOUND) - .with_header(fastly::http::header::CACHE_CONTROL, "private, no-store") - .with_body_text_plain("unknown request id") - } - } - } - ``` - -- [ ] **Step 3: Compile check** - - Run: `cargo check --workspace` - Expected: clean - -- [ ] **Step 4: Run tests** - - Run: `cargo test --workspace` - Expected: all pass - -- [ ] **Step 5: Commit** - - ```bash - git add crates/trusted-server-adapter-fastly/src/main.rs - git commit -m "Add /ts-bids endpoint with long-poll semantics; serves bid_cache results by request_id" - ``` - ---- - -## Task 11: GPT head injector — emit `__tsAdInit` with `/ts-bids` fetch +## Task 9: GPT head injector — emit `__tsAdInit` with synchronous bid read **Files:** - Modify: `crates/trusted-server-core/src/integrations/gpt.rs` -> **Critical:** The `__tsAdInit` function MUST fetch `/ts-bids?rid=` — it must NOT read from `window.__ts_bids` (which is never set). The `window.__ts_request_id` global (injected at head-open by Task 9) supplies the RID. +> **Critical:** `__tsAdInit` reads `window.__ts_bids` **synchronously** — no fetch, no Promise. `window.__ts_bids` is already on the page (injected before ``) when `__tsAdInit` runs (it executes post-DCL, after `` is received). Both `nurl` and `burl` fire client-side from `slotRenderEnded`; neither is fired server-side. - [ ] **Step 1: Write failing test** ```rust #[test] - fn head_inserts_includes_ts_ad_init_with_ts_bids_fetch() { + fn head_inserts_includes_ts_ad_init_with_synchronous_bids_read() { let config = test_config(); let integration = GptIntegration::new(config); let ctx = make_test_context(); let inserts = integration.head_inserts(&ctx); let combined = inserts.join(""); assert!(combined.contains("__tsAdInit"), "should define __tsAdInit"); - assert!(combined.contains("/ts-bids"), "should fetch from /ts-bids endpoint"); - assert!(combined.contains("__ts_request_id"), "should use __ts_request_id for rid"); - assert!(combined.contains("bidsPromise"), "should use bidsPromise pattern"); + assert!(combined.contains("window.__ts_bids"), "should read window.__ts_bids synchronously"); + assert!(combined.contains("ts_initial"), "should set ts_initial sentinel"); assert!(combined.contains("slotRenderEnded"), "should register slotRenderEnded"); - assert!(combined.contains("sendBeacon"), "should fire burl via sendBeacon"); - assert!(!combined.contains("__ts_bids"), "must NOT read window.__ts_bids — bids come from /ts-bids fetch"); + assert!(combined.contains("sendBeacon"), "should fire nurl and burl via sendBeacon"); + assert!(combined.contains("nurl"), "should fire nurl on confirmed render"); + assert!(!combined.contains("/ts-bids"), "must NOT fetch /ts-bids — bids are inline on the page"); + assert!(!combined.contains("bidsPromise"), "must NOT use bidsPromise — bids are synchronous"); + assert!(!combined.contains("__ts_request_id"), "must NOT reference request_id — no longer used"); } ``` Run: `cargo test -p trusted-server-core integrations::gpt` - Expected: FAIL — `__tsAdInit` not defined / assertion on `/ts-bids` string fails if old version present + Expected: FAIL - [ ] **Step 2: Replace `head_inserts()` in gpt.rs** @@ -1771,42 +1434,39 @@ The `/ts-bids` endpoint is the client's fetch target for bid results. It long-po "" .to_string(), - // __tsAdInit: fetches /ts-bids for bid targeting, then drives GPT. - // window.__ts_ad_slots and window.__ts_request_id are injected at head-open by TS. - // bidsPromise resolves concurrently with page rendering — never blocks FCP. + // __tsAdInit: reads window.__ts_bids synchronously (injected before ). + // No fetch, no Promise. Executes post-DCL when has already arrived. + // Both nurl and burl fire client-side from slotRenderEnded — never server-side. + // Note: window.__tsjs_installGptShim above is an EXISTING function in the + // tsjs-core bundle that stubs googletag.cmd before the real GPT loads. concat!( "" @@ -1825,18 +1485,18 @@ The `/ts-bids` endpoint is the client's fetch target for bid results. It long-po ```bash git add crates/trusted-server-core/src/integrations/gpt.rs - git commit -m "Emit __tsAdInit with /ts-bids fetch pattern from GPT head injector" + git commit -m "Emit __tsAdInit with synchronous window.__ts_bids read; nurl+burl from slotRenderEnded" ``` --- -## Task 12: `gpt/index.ts` — TypeScript `__tsAdInit` with `/ts-bids` fetch +## Task 10: `gpt/index.ts` — TypeScript `__tsAdInit` with slim-Prebid lazy loader **Files:** - Modify: `crates/js/lib/src/integrations/gpt/index.ts` -The TypeScript version mirrors the Rust inline string from Task 11. It uses the `bidsPromise` pattern — fetching `/ts-bids` concurrently with GPT slot definition. +The TypeScript version mirrors the Rust inline string from Task 9 and adds the lazy slim-Prebid loader. Slim-Prebid loads post-`window.load` and handles two things: refresh auctions (via existing GPT refresh triggers) and userID module warm-up to enrich the EC graph for the next request. - [ ] **Step 1: Write failing tests** @@ -1848,16 +1508,16 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the describe('installTsAdInit', () => { beforeEach(() => { delete (window as any).__ts_ad_slots - delete (window as any).__ts_request_id + delete (window as any).__ts_bids delete (window as any).__tsAdInit }) - it('fetches /ts-bids with request_id and applies bid targeting before refresh', async () => { + it('reads window.__ts_bids synchronously and applies bid targeting before refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('atf'), - getTargeting: vi.fn().mockReturnValue([]), + getTargeting: vi.fn().mockReturnValue(['abc']), } const mockPubads = { enableSingleRequest: vi.fn(), @@ -1879,71 +1539,94 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the targeting: { pos: 'atf' }, }, ] - ;(window as any).__ts_request_id = 'test-rid-123' - - const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ - ok: true, - json: async () => ({ - atf: { - hb_pb: '1.00', - hb_bidder: 'kargo', - hb_adid: 'abc', - burl: 'https://ssp/bill', - }, - }), - } as Response) + ;(window as any).__ts_bids = { + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + nurl: 'https://ssp/win', + burl: 'https://ssp/bill', + }, + } + + const fetchSpy = vi.spyOn(global, 'fetch') const { installTsAdInit } = await import('./index') installTsAdInit() - await (window as any).__tsAdInit() + ;(window as any).__tsAdInit() - expect(fetchSpy).toHaveBeenCalledWith( - expect.stringContaining('/ts-bids?rid=test-rid-123'), - expect.objectContaining({ credentials: 'omit' }) - ) + expect(fetchSpy).not.toHaveBeenCalled() expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00') expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'kargo') + expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1') expect(mockPubads.refresh).toHaveBeenCalled() fetchSpy.mockRestore() }) - it('calls refresh with empty bids when fetch fails', async () => { + it('fires both nurl and burl via sendBeacon on slotRenderEnded when our bid won', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) + let capturedListener: ((e: any) => void) | undefined + + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('atf'), + getTargeting: vi.fn().mockReturnValue(['abc']), + } const mockPubads = { enableSingleRequest: vi.fn(), - addEventListener: vi.fn(), refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: any) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn + }), } ;(window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, - defineSlot: vi.fn().mockReturnValue({ - addService: vi.fn().mockReturnThis(), - setTargeting: vi.fn().mockReturnThis(), - }), + defineSlot: vi.fn().mockReturnValue(mockSlot), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), } - ;(window as any).__ts_ad_slots = [] - ;(window as any).__ts_request_id = 'rid-fail' - - vi.spyOn(global, 'fetch').mockRejectedValue(new Error('network error')) + ;(window as any).__ts_ad_slots = [ + { + id: 'atf', + gam_unit_path: '/123/atf', + div_id: 'atf', + formats: [[300, 250]], + targeting: {}, + }, + ] + ;(window as any).__ts_bids = { + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + nurl: 'https://ssp/win', + burl: 'https://ssp/bill', + }, + } const { installTsAdInit } = await import('./index') installTsAdInit() - await (window as any).__tsAdInit() + ;(window as any).__tsAdInit() - expect(mockPubads.refresh).toHaveBeenCalled() + expect(capturedListener).toBeDefined() + capturedListener!({ isEmpty: false, slot: mockSlot }) + + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/win') + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill') + beaconSpy.mockRestore() }) - it('fires burl via sendBeacon on slotRenderEnded when our bid won', async () => { + it('does not fire nurl/burl when bid did not win GAM line item', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) let capturedListener: ((e: any) => void) | undefined - const mockSlot = { + const mockSlotNoMatch = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('atf'), - getTargeting: vi.fn().mockReturnValue(['abc']), + getTargeting: vi.fn().mockReturnValue(['OTHER_BID_ID']), } const mockPubads = { enableSingleRequest: vi.fn(), @@ -1954,7 +1637,7 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the } ;(window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, - defineSlot: vi.fn().mockReturnValue(mockSlot), + defineSlot: vi.fn().mockReturnValue(mockSlotNoMatch), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), } @@ -1967,43 +1650,58 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the targeting: {}, }, ] - ;(window as any).__ts_request_id = 'rid-burl-test' - - vi.spyOn(global, 'fetch').mockResolvedValue({ - ok: true, - json: async () => ({ - atf: { - hb_pb: '1.00', - hb_bidder: 'kargo', - hb_adid: 'abc', - burl: 'https://ssp/bill', - }, - }), - } as Response) + ;(window as any).__ts_bids = { + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + nurl: 'https://ssp/win', + burl: 'https://ssp/bill', + }, + } const { installTsAdInit } = await import('./index') installTsAdInit() - await (window as any).__tsAdInit() + ;(window as any).__tsAdInit() + capturedListener!({ isEmpty: false, slot: mockSlotNoMatch }) - // Trigger slotRenderEnded — slot has our winning hb_adid - expect(capturedListener).toBeDefined() - capturedListener!({ - isEmpty: false, - slot: mockSlot, - }) - - expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill') + expect(beaconSpy).not.toHaveBeenCalled() beaconSpy.mockRestore() }) + + it('calls refresh even when __ts_bids is empty (graceful fallback)', () => { + const mockPubads = { + enableSingleRequest: vi.fn(), + addEventListener: vi.fn(), + refresh: vi.fn(), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue({ + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + }), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [] + ;(window as any).__ts_bids = {} + + const { installTsAdInit } = require('./index') + installTsAdInit() + ;(window as any).__tsAdInit() + + expect(mockPubads.refresh).toHaveBeenCalled() + }) }) ``` Run: `cd crates/js/lib && npx vitest run` - Expected: FAIL — `installTsAdInit` not exported or fetches wrong endpoint + Expected: FAIL — `installTsAdInit` not defined or assertions fail -- [ ] **Step 2: Add `installTsAdInit` to `index.ts`** +- [ ] **Step 2: Implement `installTsAdInit` in `index.ts`** - Add to `crates/js/lib/src/integrations/gpt/index.ts`: + Replace the old `/ts-bids` fetch implementation with: ```typescript interface TsAdSlot { @@ -2018,38 +1716,30 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the hb_pb?: string hb_bidder?: string hb_adid?: string + nurl?: string burl?: string } type TsWindow = Window & { __ts_ad_slots?: TsAdSlot[] - __ts_request_id?: string + __ts_bids?: Record __tsAdInit?: () => void } /** * Install `window.__tsAdInit`. * - * Reads `window.__ts_ad_slots` and `window.__ts_request_id` (both injected by - * the edge at `` open). Fetches bid results from `/ts-bids?rid=` - * concurrently with GPT slot definition. Applies targeting and calls `refresh()` - * after the fetch resolves. Registers `slotRenderEnded` to fire `burl` via - * `sendBeacon` when our specific Prebid bid wins the GAM line item match. + * Reads `window.__ts_ad_slots` (injected at head-open) and `window.__ts_bids` + * (injected before ) synchronously — no fetch, no Promise. Applies bid + * targeting to GPT slots, sets the `ts_initial` sentinel, registers + * `slotRenderEnded` to fire both nurl and burl via sendBeacon when our + * specific Prebid bid wins the GAM line item match, then calls refresh(). */ export function installTsAdInit(): void { const w = window as TsWindow w.__tsAdInit = function () { const slots = w.__ts_ad_slots ?? [] - const rid = w.__ts_request_id - - const bidsPromise: Promise> = rid - ? fetch(`/ts-bids?rid=${encodeURIComponent(rid)}`, { - credentials: 'omit', - }) - .then((r) => (r.ok ? r.json() : {})) - .catch(() => ({})) - : Promise.resolve({}) - + const bids = w.__ts_bids ?? {} const g = (window as GptWindow).googletag if (!g) return @@ -2066,6 +1756,11 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v) ) + const bid = bids[slot.id] ?? {} + ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + }) + gptSlot.setTargeting('ts_initial', '1') return { id: slot.id, gptSlot } }) .filter(Boolean) as Array<{ @@ -2076,153 +1771,86 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the g.pubads().enableSingleRequest() g.enableServices() - bidsPromise.then((bids) => { - gptSlots.forEach(({ id, gptSlot }) => { - const bid = bids[id] ?? {} - ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { - if (bid[key]) gptSlot.setTargeting(key, bid[key]!) - }) - }) - - g.pubads().addEventListener?.('slotRenderEnded', (event: any) => { - const slotId: string = event.slot?.getSlotElementId?.() ?? '' - const bid = bids[slotId] ?? {} - if ( - !event.isEmpty && - bid.burl && - event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid - ) { - navigator.sendBeacon(bid.burl) - } - }) - - g.pubads().refresh() + g.pubads().addEventListener?.('slotRenderEnded', (event: any) => { + const slotId: string = event.slot?.getSlotElementId?.() ?? '' + const bid = bids[slotId] ?? {} + const ourBidWon = + !event.isEmpty && + bid.hb_adid && + event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + if (ourBidWon) { + if (bid.nurl) navigator.sendBeacon(bid.nurl) + if (bid.burl) navigator.sendBeacon(bid.burl) + } }) + + g.pubads().refresh() }) } } ``` - Call `installTsAdInit()` from the integration's initialization path. +- [ ] **Step 3: Add lazy slim-Prebid loader (post-`window.load`)** -- [ ] **Step 3: Run JS tests** + After `installTsAdInit`, add: - Run: `cd crates/js/lib && npx vitest run` - Expected: new tests pass - -- [ ] **Step 4: Build JS bundle** - - Run: `cd crates/js/lib && node build-all.mjs` - Expected: clean build - -- [ ] **Step 5: Commit** - - ```bash - git add crates/js/lib/src/integrations/gpt/ - git commit -m "Add installTsAdInit with /ts-bids fetch pattern and slotRenderEnded burl firing" - ``` - ---- - -## Task 13: `nurl` fire-and-forget - -**Files:** - -- Modify: `crates/trusted-server-core/src/integrations/prebid.rs` -- Modify: `crates/trusted-server-core/src/publisher.rs` - -- [ ] **Step 1: Write failing test** - - ```rust - #[test] - fn prebid_config_fire_nurl_defaults_to_true() { - let config = PrebidConfig::default(); - assert!(config.fire_nurl_at_edge, "should fire nurl at edge by default"); + ```typescript + /** + * Register the slim-Prebid lazy loader. Fires after window.load — off the + * critical path. slim-Prebid handles refresh auctions and userID module + * warm-up (ID5, sharedID, LiveRamp ATS, Lockr). It skips initial-render slots + * (ts_initial=1) and registers as the GPT refresh handler for scroll/sticky auctions. + * + * Phase 1: no-op unless window.__tsjs_slim_prebid_url is set (it won't be until + * the slim-Prebid bundle build target ships in a later phase). + */ + export function installSlimPrebidLoader(): void { + const url = (window as any).__tsjs_slim_prebid_url as string | undefined + if (!url) return + window.addEventListener('load', () => { + const script = document.createElement('script') + script.src = url + script.defer = true + document.head.appendChild(script) + }) } ``` - Run: `cargo test -p trusted-server-core integrations::prebid` - Expected: FAIL - -- [ ] **Step 2: Add `fire_nurl_at_edge` to `PrebidConfig`** + Call `installTsAdInit()` from the integration's existing initialization path — wherever the module's init function runs at page load (look for the existing `init()` or module-level call that sets up the GPT integration). Add: - ```rust - #[serde(default = "default_fire_nurl_at_edge")] - pub fire_nurl_at_edge: bool, - ``` - - ```rust - fn default_fire_nurl_at_edge() -> bool { true } - ``` - -- [ ] **Step 3: Fire nurls in publisher.rs after bid_cache.put()** - - After the `bid_cache.put(...)` call (Task 9 Step 3), add: - - ```rust - if let Some(ref result) = auction_result { - fire_winning_nurls(result, settings); - } + ```typescript + // In the integration's init / module entry point: + installTsAdInit() ``` - Add helper: - - ```rust - fn fire_winning_nurls( - result: &crate::auction::orchestrator::OrchestrationResult, - settings: &Settings, - ) { - use crate::backend::BackendConfig; - - let fire_nurl = settings - .integrations - .get_typed::("prebid") - .map(|c| c.fire_nurl_at_edge) - .unwrap_or(true); + `window.__tsAdInit()` itself is called by `__tsAdInit` being invoked from the `"); @@ -2289,7 +1914,7 @@ Tests use `pub(crate)` helpers from Task 9 directly. } #[test] - fn bid_map_uses_price_bucket_and_ad_id() { + fn bid_map_includes_nurl_and_burl() { let mut winning_bids = HashMap::new(); winning_bids.insert("atf_sidebar_ad".to_string(), Bid { slot_id: "atf_sidebar_ad".to_string(), @@ -2298,46 +1923,60 @@ Tests use `pub(crate)` helpers from Task 9 directly. creative: None, adomain: None, bidder: "kargo".to_string(), - width: 300, height: 250, + width: 300, + height: 250, + nurl: Some("https://ssp/win".to_string()), + burl: Some("https://ssp/bill".to_string()), + ad_id: Some("abc123".to_string()), + metadata: Default::default(), + }); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let entry = map.get("atf_sidebar_ad").expect("should have bid entry"); + assert_eq!(entry.get("hb_pb").and_then(|v| v.as_str()), Some("2.50")); + assert_eq!(entry.get("hb_bidder").and_then(|v| v.as_str()), Some("kargo")); + assert_eq!(entry.get("hb_adid").and_then(|v| v.as_str()), Some("abc123")); + assert_eq!(entry.get("nurl").and_then(|v| v.as_str()), Some("https://ssp/win")); + assert_eq!(entry.get("burl").and_then(|v| v.as_str()), Some("https://ssp/bill")); + } + + #[test] + fn bid_map_excludes_slot_when_price_is_none() { + let mut winning_bids = HashMap::new(); + winning_bids.insert("no-price-slot".to_string(), Bid { + slot_id: "no-price-slot".to_string(), + price: None, + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, + height: 250, nurl: None, - burl: Some("https://ssp.example/billing?id=abc123".to_string()), - ad_id: Some("prebid-uuid-abc123".to_string()), - metadata: HashMap::new(), + burl: None, + ad_id: None, + metadata: Default::default(), }); - let bid_map = build_bid_map(&winning_bids, PriceGranularity::Dense); - let slot_bids = bid_map.get("atf_sidebar_ad").expect("should have slot bids"); - assert_eq!( - slot_bids.get("hb_pb").and_then(|v| v.as_str()), - Some("2.53"), - "should bucket 2.53 as 2.53 (dense)" - ); - assert_eq!( - slot_bids.get("hb_bidder").and_then(|v| v.as_str()), - Some("kargo"), - "should include bidder" - ); - assert_eq!( - slot_bids.get("hb_adid").and_then(|v| v.as_str()), - Some("prebid-uuid-abc123"), - "should use ad_id not creative markup" - ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + assert!(map.is_empty(), "slot with no price should be excluded from bid map"); } #[test] - fn html_escape_neutralizes_xss_in_json() { - let malicious = r#"{"zone":""), "should escape "); - assert!(escaped.contains("\\u003c"), "should unicode-escape <"); - assert!(escaped.contains("\\u003e"), "should unicode-escape >"); + fn bids_script_is_xss_safe() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + let script = build_bids_script(&map); + let inner = script + .trim_start_matches(""); + assert!(!inner.contains('<'), "no unescaped < in bids script"); + assert!(!inner.contains('>'), "no unescaped > in bids script"); } #[test] - fn url_matching_end_to_end() { - let file = CreativeOpportunitiesFile { slots: vec![make_slot()] }; - assert_eq!(match_slots(&file.slots, "/2024/01/my-article").len(), 1, "should match article"); - assert_eq!(match_slots(&file.slots, "/about").len(), 0, "should not match /about"); - assert_eq!(match_slots(&file.slots, "/").len(), 0, "should not match root"); + fn html_escape_encodes_special_chars() { + assert_eq!(html_escape_for_script("`. + /// Injected at `` open. `None` when no slots matched. + pub ad_slots_script: Option, + /// Shared auction result — written by auction task before HTML processing begins. + /// Handler reads this in `el.on_end_tag()` on the body element. + /// `None` means no auction ran; inject empty `__ts_bids = {}` as fallback. + pub ad_bids_state: std::sync::Arc>>, } impl HtmlProcessorConfig { @@ -151,6 +158,8 @@ impl HtmlProcessorConfig { request_host: request_host.to_string(), request_scheme: request_scheme.to_string(), integrations: integrations.clone(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), } } } @@ -230,6 +239,8 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let injected_tsjs = Rc::new(Cell::new(false)); let integration_registry = config.integrations.clone(); let script_rewriters = integration_registry.script_rewriters(); + let ad_slots_script = config.ad_slots_script.clone(); + let ad_bids_state = config.ad_bids_state.clone(); let mut element_content_handlers = vec![ // Inject unified tsjs bundle once at the start of @@ -238,9 +249,14 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let integrations = integration_registry.clone(); let patterns = patterns.clone(); let document_state = document_state.clone(); + let ad_slots_script = ad_slots_script.clone(); move |el| { if !injected_tsjs.get() { let mut snippet = String::new(); + // Inject ad slots script first so it appears before tsjs bundle. + if let Some(ref slots_script) = ad_slots_script { + snippet.push_str(slots_script); + } let ctx = IntegrationHtmlContext { request_host: &patterns.request_host, request_scheme: &patterns.request_scheme, @@ -265,6 +281,30 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso Ok(()) } }), + // Inject __ts_bids before via end_tag_handlers. + element!("body", { + let state = ad_bids_state.clone(); + move |el| { + let state = state.clone(); + if let Some(handlers) = el.end_tag_handlers() { + let handler: EndTagHandler<'static> = + Box::new(move |end_tag: &mut EndTag<'_>| { + let script_guard = state.read().expect("should read bid state"); + let bids_script = match &*script_guard { + Some(s) => s.clone(), + None => { + r#""# + .to_string() + } + }; + end_tag.before(&bids_script, ContentType::Html); + Ok(()) + }); + handlers.push(handler); + } + Ok(()) + } + }), // Replace URLs in href attributes element!("[href]", { let patterns = patterns.clone(); @@ -540,6 +580,8 @@ mod tests { request_host: "test.example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), } } @@ -1185,4 +1227,85 @@ mod tests { "should contain post-processor mutation" ); } + + #[test] + fn injects_ad_slots_at_head_open() { + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: Some( + r#""#.to_string(), + ), + ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk( + b"Tcontent", + true, + ) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + html.contains("window.__ts_ad_slots"), + "should inject ad slots at head-open" + ); + assert!( + !html.contains("__ts_request_id"), + "must NOT inject request_id" + ); + } + + #[test] + fn injects_ts_bids_before_body_close() { + let bids_script = + r#""#; + let state = std::sync::Arc::new(std::sync::RwLock::new(Some(bids_script.to_string()))); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + html.contains("window.__ts_bids"), + "should inject bids before " + ); + let bids_pos = html + .find("window.__ts_bids") + .expect("bids should be in output"); + let body_close_pos = html.find("").expect(" should be in output"); + assert!(bids_pos < body_close_pos, "bids must appear before "); + } + + #[test] + fn injects_empty_ts_bids_when_state_is_none() { + let state = std::sync::Arc::new(std::sync::RwLock::new(None)); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + html.contains("__ts_bids=JSON.parse(\"{}\")"), + "should inject empty bids on None state" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 8b55493be..ffad78921 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -853,6 +853,14 @@ impl IntegrationRegistry { .collect() } + #[cfg(test)] + #[must_use] + pub fn empty_for_tests() -> Self { + Self { + inner: Arc::new(IntegrationRegistryInner::default()), + } + } + #[cfg(test)] #[must_use] pub fn from_rewriters( From 8b9500cf7c5d83d4d7bf97910ddb414651ec704d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 5 May 2026 19:47:16 +0530 Subject: [PATCH 016/395] Convert handle_publisher_request to async; body-inject __ts_bids; eligibility gates; max-age=0 - Make handle_publisher_request async; add orchestrator and slots_file params - Dispatch origin request with send_async before running auction in parallel - Gate auction on GET, no prefetch, no bot, matched slots, TCF purpose-1 consent - Run server-side auction and write bucketed bids to ad_bids_state Arc - Compute ad_slots_script after response headers; set Cache-Control: private, max-age=0 - Fix Stream arm to thread actual ad_slots_script and ad_bids_state through - Add build_auction_request, build_bid_map, build_bids_script, build_ad_slots_script helpers - Update route_tests.rs to pass empty slots_file to route_request --- .../src/route_tests.rs | 6 + crates/trusted-server-core/src/publisher.rs | 249 +++++++++++++++++- 2 files changed, 243 insertions(+), 12 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/route_tests.rs b/crates/trusted-server-adapter-fastly/src/route_tests.rs index 0fd0113f8..06336a9b1 100644 --- a/crates/trusted-server-adapter-fastly/src/route_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/route_tests.rs @@ -184,6 +184,8 @@ fn configured_missing_consent_store_only_breaks_consent_routes() { let orchestrator = build_orchestrator(&settings).expect("should build auction orchestrator"); let integration_registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); + let slots_file = + trusted_server_core::creative_opportunities::CreativeOpportunitiesFile::default(); let discovery_req = Request::get("https://test.com/.well-known/trusted-server.json"); let discovery_services = test_runtime_services(&discovery_req); @@ -192,6 +194,7 @@ fn configured_missing_consent_store_only_breaks_consent_routes() { &orchestrator, &integration_registry, &discovery_services, + &slots_file, discovery_req, )) .expect("should route discovery request"); @@ -208,6 +211,7 @@ fn configured_missing_consent_store_only_breaks_consent_routes() { &orchestrator, &integration_registry, &admin_services, + &slots_file, admin_req, )) .expect("should route admin request"); @@ -224,6 +228,7 @@ fn configured_missing_consent_store_only_breaks_consent_routes() { &orchestrator, &integration_registry, &auction_services, + &slots_file, auction_req, )) .expect("should return an error response for auction requests"); @@ -240,6 +245,7 @@ fn configured_missing_consent_store_only_breaks_consent_routes() { &orchestrator, &integration_registry, &publisher_services, + &slots_file, publisher_req, )) .expect("should return an error response for publisher fallback"); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 5bcef6941..4037bdf8a 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -12,11 +12,14 @@ //! content-rewriting concern. use std::io::Write; +use std::sync::{Arc, RwLock}; use error_stack::{Report, ResultExt}; use fastly::http::{header, StatusCode}; use fastly::{Body, Request, Response}; +use crate::auction::orchestrator::AuctionOrchestrator; +use crate::auction::types::{AuctionContext, AuctionRequest, Bid, PublisherInfo, SiteInfo, UserInfo}; use crate::backend::BackendConfig; use crate::consent::{allows_ec_creation, build_consent_context, ConsentPipelineInput}; use crate::constants::{COOKIE_TS_EC, HEADER_X_COMPRESS_HINT, HEADER_X_TS_EC}; @@ -26,6 +29,7 @@ use crate::error::TrustedServerError; use crate::http_util::{serve_static_with_etag, RequestInfo}; use crate::integrations::IntegrationRegistry; use crate::platform::RuntimeServices; +use crate::price_bucket::price_bucket; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, StreamingPipeline}; @@ -182,6 +186,8 @@ struct ProcessResponseParams<'a> { settings: &'a Settings, content_type: &'a str, integration_registry: &'a IntegrationRegistry, + ad_slots_script: Option<&'a str>, + ad_bids_state: &'a Arc>>, } /// Process response body through the streaming pipeline. @@ -224,6 +230,8 @@ fn process_response_streaming( params.request_scheme, params.settings, params.integration_registry, + params.ad_slots_script.map(str::to_string), + params.ad_bids_state.clone(), )?; StreamingPipeline::new(config, processor).process(body, output)?; } else if is_rsc_flight { @@ -252,18 +260,21 @@ fn create_html_stream_processor( origin_host: &str, request_host: &str, request_scheme: &str, - settings: &Settings, + _settings: &Settings, integration_registry: &IntegrationRegistry, + ad_slots_script: Option, + ad_bids_state: Arc>>, ) -> Result> { use crate::html_processor::{create_html_processor, HtmlProcessorConfig}; - let config = HtmlProcessorConfig::from_settings( - settings, - integration_registry, - origin_host, - request_host, - request_scheme, - ); + let config = HtmlProcessorConfig { + origin_host: origin_host.to_string(), + request_host: request_host.to_string(), + request_scheme: request_scheme.to_string(), + integrations: integration_registry.clone(), + ad_slots_script, + ad_bids_state, + }; Ok(create_html_processor(config)) } @@ -392,6 +403,8 @@ pub struct OwnedProcessResponseParams { pub(crate) request_host: String, pub(crate) request_scheme: String, pub(crate) content_type: String, + pub(crate) ad_slots_script: Option, + pub(crate) ad_bids_state: Arc>>, } /// Stream the publisher response body through the processing pipeline. @@ -420,6 +433,8 @@ pub fn stream_publisher_body( settings, content_type: ¶ms.content_type, integration_registry, + ad_slots_script: params.ad_slots_script.as_deref(), + ad_bids_state: ¶ms.ad_bids_state, }; process_response_streaming(body, output, &borrowed) } @@ -441,10 +456,12 @@ pub fn stream_publisher_body( /// /// Returns a [`TrustedServerError`] if the proxy request fails or the /// origin backend is unreachable. -pub fn handle_publisher_request( +pub async fn handle_publisher_request( settings: &Settings, integration_registry: &IntegrationRegistry, services: &RuntimeServices, + orchestrator: &AuctionOrchestrator, + slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, mut req: Request, ) -> Result> { log::debug!("Proxying request to publisher_origin"); @@ -520,14 +537,105 @@ pub fn handle_publisher_request( backend_name, settings.publisher.origin_url ); + + let request_path = req.get_path().to_string(); + let is_get = req.get_method() == fastly::http::Method::GET; + + let is_prefetch = req.get_header_str("sec-purpose") + .map_or(false, |v| v.contains("prefetch")) + || req.get_header_str("purpose") + .map_or(false, |v| v.contains("prefetch")); + + let user_agent = req.get_header_str("user-agent").unwrap_or(""); + let is_bot = ["Googlebot", "Bingbot", "AhrefsBot", "SemrushBot", "DotBot"] + .iter() + .any(|bot| user_agent.contains(bot)); + + let matched_slots: Vec<_> = if settings.creative_opportunities.is_some() && is_get { + crate::creative_opportunities::match_slots(&slots_file.slots, &request_path) + .into_iter() + .cloned() + .collect() + } else { + Vec::new() + }; + + let consent_allows_auction = consent_context + .tcf + .as_ref() + .map_or(false, |tcf| tcf.has_purpose_consent(1)); + + let should_run_auction = is_get + && !is_prefetch + && !is_bot + && !matched_slots.is_empty() + && consent_allows_auction; + + let auction_timeout_ms = settings + .creative_opportunities + .as_ref() + .and_then(|co| co.auction_timeout_ms) + .unwrap_or(settings.auction.timeout_ms); + + let ad_bids_state: Arc>> = Arc::new(RwLock::new(None)); + // Only advertise encodings the rewrite pipeline can decode and re-encode. restrict_accept_encoding(&mut req); req.set_header("host", &origin_host); - let mut response = req - .send(&backend_name) + let pending_origin = req + .send_async(&backend_name) .change_context(TrustedServerError::Proxy { - message: "Failed to proxy request to origin".to_string(), + message: "Failed to dispatch async origin request".to_string(), + })?; + + let auction_result = if should_run_auction { + let co_config = settings.creative_opportunities.as_ref() + .expect("should be present when should_run_auction is true"); + let auction_request = build_auction_request( + &matched_slots, + &ec_id, + &consent_context, + &request_info, + co_config, + ); + let placeholder_req = fastly::Request::get("https://placeholder.invalid/"); + let auction_context = AuctionContext { + settings, + request: &placeholder_req, + client_info: services.client_info(), + timeout_ms: auction_timeout_ms, + provider_responses: None, + services, + }; + match orchestrator.run_auction(&auction_request, &auction_context, services).await { + Ok(result) => Some(result), + Err(e) => { + log::warn!("server-side auction failed, proceeding without bids: {e:?}"); + None + } + } + } else { + None + }; + + if should_run_auction { + let co_config = settings.creative_opportunities.as_ref() + .expect("should be present"); + let empty: std::collections::HashMap = + std::collections::HashMap::new(); + let winning_bids = auction_result.as_ref() + .map(|r| &r.winning_bids) + .unwrap_or(&empty); + let bid_map = build_bid_map(winning_bids, co_config.price_granularity); + let bids_script = build_bids_script(&bid_map); + *ad_bids_state.write().expect("should write bid state") = Some(bids_script); + } + + let mut response = pending_origin + .wait() + .change_context(TrustedServerError::Proxy { + message: "Failed to await origin response".to_string(), })?; log::debug!("Response headers:"); @@ -535,6 +643,22 @@ pub fn handle_publisher_request( log::debug!(" {}: {:?}", name, value); } + let ad_slots_script = if let Some(co_config) = &settings.creative_opportunities { + if !matched_slots.is_empty() { + Some(build_ad_slots_script(&matched_slots, co_config)) + } else { + None + } + } else { + None + }; + + if ad_slots_script.is_some() { + response.set_header(header::CACHE_CONTROL, "private, max-age=0"); + response.remove_header("surrogate-control"); + response.remove_header("fastly-surrogate-control"); + } + // Set EC ID / cookie headers BEFORE body processing. // These are body-independent (computed from request cookies + consent). apply_ec_headers( @@ -623,6 +747,8 @@ pub fn handle_publisher_request( request_host: request_host.to_string(), request_scheme: request_scheme.to_string(), content_type, + ad_slots_script: ad_slots_script.clone(), + ad_bids_state: ad_bids_state.clone(), }, }) } @@ -642,6 +768,8 @@ pub fn handle_publisher_request( settings, content_type: &content_type, integration_registry, + ad_slots_script: ad_slots_script.as_deref(), + ad_bids_state: &ad_bids_state, }; let mut output = Vec::new(); process_response_streaming(body, &mut output, ¶ms)?; @@ -654,6 +782,93 @@ pub fn handle_publisher_request( } } +/// Build an [`AuctionRequest`] from matched creative opportunity slots. +pub(crate) fn build_auction_request( + matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + ec_id: &str, + consent_context: &crate::consent::ConsentContext, + request_info: &crate::http_util::RequestInfo, + co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, +) -> AuctionRequest { + let slots = matched_slots + .iter() + .map(|s| s.to_ad_slot(&co_config.gam_network_id)) + .collect(); + AuctionRequest { + id: format!("ts-{}", ec_id), + slots, + publisher: PublisherInfo { + domain: request_info.host.clone(), + page_url: None, + }, + user: UserInfo { + id: ec_id.to_string(), + fresh_id: ec_id.to_string(), + consent: Some(consent_context.clone()), + }, + device: None, + site: Some(SiteInfo { + domain: request_info.host.clone(), + page: String::new(), + }), + context: std::collections::HashMap::new(), + } +} + +/// Build a price-bucketed bid map from winning bids. +/// +/// Returns a map of slot ID → bucketed CPM string. +pub(crate) fn build_bid_map( + winning_bids: &std::collections::HashMap, + granularity: crate::price_bucket::PriceGranularity, +) -> std::collections::HashMap { + winning_bids + .iter() + .filter_map(|(slot_id, bid)| { + bid.price.map(|cpm| { + let bucket = price_bucket(cpm, granularity); + (slot_id.clone(), bucket) + }) + }) + .collect() +} + +/// Build the `__ts_bids` inline script content from a bucketed bid map. +pub(crate) fn build_bids_script(bid_map: &std::collections::HashMap) -> String { + let entries: Vec = bid_map + .iter() + .map(|(slot_id, bucket)| format!("\"{}\":\"{}\"", slot_id, bucket)) + .collect(); + format!("window.__ts_bids={{{}}};", entries.join(",")) +} + +/// Build the `__ts_ad_slots` inline script content from matched slots. +pub(crate) fn build_ad_slots_script( + matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, +) -> String { + let entries: Vec = matched_slots + .iter() + .map(|slot| { + let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); + let div_id = slot.resolved_div_id(); + let formats: Vec = slot + .formats + .iter() + .map(|f| format!("[{},{}]", f.width, f.height)) + .collect(); + format!( + "{{\"id\":\"{}\",\"div\":\"{}\",\"path\":\"{}\",\"sizes\":[{}]}}", + slot.id, + div_id, + gam_path, + formats.join(",") + ) + }) + .collect(); + format!("window.__ts_ad_slots=[{}];", entries.join(",")) +} + /// Whether the content type requires processing (URL rewriting, HTML injection). /// /// Text-based and JavaScript/JSON responses are processable; binary types @@ -1366,6 +1581,8 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(RwLock::new(None)), }; let mut output = Vec::new(); @@ -1407,6 +1624,8 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(RwLock::new(None)), }; let mut output = Vec::new(); @@ -1439,6 +1658,8 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(RwLock::new(None)), }; let bogus_body = Body::from(b"not gzip".to_vec()); @@ -1538,6 +1759,8 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(RwLock::new(None)), }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -1588,6 +1811,8 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(RwLock::new(None)), }; let mut output = Vec::new(); From 9cdbb36fd7bb62212258a433c2764f07eb8f7e54 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 5 May 2026 20:05:59 +0530 Subject: [PATCH 017/395] Emit __tsAdInit with synchronous window.__ts_bids read; nurl+burl from slotRenderEnded --- .../src/integrations/gpt.rs | 89 +++++++++++++++++-- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 40bcf7f2c..796d633e1 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -438,13 +438,42 @@ impl IntegrationHeadInjector for GptIntegration { } fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - // Set the enable flag and best-effort call the activation function - // registered by the GPT shim module. The bundle also auto-installs - // when it sees the pre-set flag, so this works regardless of whether - // the inline bootstrap runs before or after the TSJS bundle. vec![ - "" + "" .to_string(), + concat!( + "" + ).to_string(), ] } } @@ -1020,7 +1049,7 @@ mod tests { let inserts = integration.head_inserts(&ctx); - assert_eq!(inserts.len(), 1, "should emit exactly one head insert"); + assert_eq!(inserts.len(), 2, "should emit exactly two head inserts"); assert_eq!( inserts[0], "", @@ -1028,6 +1057,54 @@ mod tests { ); } + #[test] + fn head_inserts_includes_ts_ad_init_with_synchronous_bids_read() { + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let inserts = integration.head_inserts(&ctx); + let combined = inserts.join(""); + assert!(combined.contains("__tsAdInit"), "should define __tsAdInit"); + assert!( + combined.contains("window.__ts_bids"), + "should read window.__ts_bids synchronously" + ); + assert!( + combined.contains("ts_initial"), + "should set ts_initial sentinel" + ); + assert!( + combined.contains("slotRenderEnded"), + "should register slotRenderEnded" + ); + assert!( + combined.contains("sendBeacon"), + "should fire nurl and burl via sendBeacon" + ); + assert!( + combined.contains("nurl"), + "should fire nurl on confirmed render" + ); + assert!( + !combined.contains("/ts-bids"), + "must NOT fetch /ts-bids — bids are inline on the page" + ); + assert!( + !combined.contains("bidsPromise"), + "must NOT use bidsPromise — bids are synchronous" + ); + assert!( + !combined.contains("__ts_request_id"), + "must NOT reference request_id — no longer used" + ); + } + #[test] fn head_injector_integration_id() { let integration = GptIntegration::new(test_config()); From 6b624e3c9262cdc06b163eec4b7e18d024acdb3e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 09:32:32 +0530 Subject: [PATCH 018/395] Fix bid map shape and ad slots property names; resolve clippy errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build_bid_map now returns serde_json::Map with full bid objects (hb_pb, hb_bidder, hb_adid, nurl, burl) instead of a plain CPM string map - build_bids_script / build_ad_slots_script now emit full "# - .to_string() - } + None => r#""# + .to_string(), }; end_tag.before(&bids_script, ContentType::Html); Ok(()) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4037bdf8a..c614e5ed4 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -19,7 +19,9 @@ use fastly::http::{header, StatusCode}; use fastly::{Body, Request, Response}; use crate::auction::orchestrator::AuctionOrchestrator; -use crate::auction::types::{AuctionContext, AuctionRequest, Bid, PublisherInfo, SiteInfo, UserInfo}; +use crate::auction::types::{ + AuctionContext, AuctionRequest, Bid, PublisherInfo, SiteInfo, UserInfo, +}; use crate::backend::BackendConfig; use crate::consent::{allows_ec_creation, build_consent_context, ConsentPipelineInput}; use crate::constants::{COOKIE_TS_EC, HEADER_X_COMPRESS_HINT, HEADER_X_TS_EC}; @@ -456,6 +458,12 @@ pub fn stream_publisher_body( /// /// Returns a [`TrustedServerError`] if the proxy request fails or the /// origin backend is unreachable. +/// +/// # Panics +/// +/// Panics if `should_run_auction` is `true` but `settings.creative_opportunities` is `None`. +/// This is a logic invariant: `should_run_auction` is only set when creative opportunities +/// are configured, so this state is unreachable in practice. pub async fn handle_publisher_request( settings: &Settings, integration_registry: &IntegrationRegistry, @@ -541,10 +549,12 @@ pub async fn handle_publisher_request( let request_path = req.get_path().to_string(); let is_get = req.get_method() == fastly::http::Method::GET; - let is_prefetch = req.get_header_str("sec-purpose") - .map_or(false, |v| v.contains("prefetch")) - || req.get_header_str("purpose") - .map_or(false, |v| v.contains("prefetch")); + let is_prefetch = req + .get_header_str("sec-purpose") + .is_some_and(|v| v.contains("prefetch")) + || req + .get_header_str("purpose") + .is_some_and(|v| v.contains("prefetch")); let user_agent = req.get_header_str("user-agent").unwrap_or(""); let is_bot = ["Googlebot", "Bingbot", "AhrefsBot", "SemrushBot", "DotBot"] @@ -563,13 +573,10 @@ pub async fn handle_publisher_request( let consent_allows_auction = consent_context .tcf .as_ref() - .map_or(false, |tcf| tcf.has_purpose_consent(1)); + .is_some_and(|tcf| tcf.has_purpose_consent(1)); - let should_run_auction = is_get - && !is_prefetch - && !is_bot - && !matched_slots.is_empty() - && consent_allows_auction; + let should_run_auction = + is_get && !is_prefetch && !is_bot && !matched_slots.is_empty() && consent_allows_auction; let auction_timeout_ms = settings .creative_opportunities @@ -583,14 +590,16 @@ pub async fn handle_publisher_request( restrict_accept_encoding(&mut req); req.set_header("host", &origin_host); - let pending_origin = req - .send_async(&backend_name) - .change_context(TrustedServerError::Proxy { - message: "Failed to dispatch async origin request".to_string(), - })?; + let pending_origin = + req.send_async(&backend_name) + .change_context(TrustedServerError::Proxy { + message: "Failed to dispatch async origin request".to_string(), + })?; let auction_result = if should_run_auction { - let co_config = settings.creative_opportunities.as_ref() + let co_config = settings + .creative_opportunities + .as_ref() .expect("should be present when should_run_auction is true"); let auction_request = build_auction_request( &matched_slots, @@ -608,7 +617,10 @@ pub async fn handle_publisher_request( provider_responses: None, services, }; - match orchestrator.run_auction(&auction_request, &auction_context, services).await { + match orchestrator + .run_auction(&auction_request, &auction_context, services) + .await + { Ok(result) => Some(result), Err(e) => { log::warn!("server-side auction failed, proceeding without bids: {e:?}"); @@ -620,11 +632,13 @@ pub async fn handle_publisher_request( }; if should_run_auction { - let co_config = settings.creative_opportunities.as_ref() + let co_config = settings + .creative_opportunities + .as_ref() .expect("should be present"); - let empty: std::collections::HashMap = - std::collections::HashMap::new(); - let winning_bids = auction_result.as_ref() + let empty: std::collections::HashMap = std::collections::HashMap::new(); + let winning_bids = auction_result + .as_ref() .map(|r| &r.winning_bids) .unwrap_or(&empty); let bid_map = build_bid_map(winning_bids, co_config.price_granularity); @@ -815,58 +829,103 @@ pub(crate) fn build_auction_request( } } +/// Escape a JSON string so it is safe to embed inside a JS double-quoted string literal. +/// +/// Backslashes are doubled first (so they survive the next pass), then +/// double-quotes are escaped so they do not terminate the JS string. +/// The result is always valid to write as `JSON.parse("…")`. +fn html_escape_for_script(s: &str) -> String { + s.replace('\\', "\\\\").replace('"', "\\\"") +} + /// Build a price-bucketed bid map from winning bids. /// -/// Returns a map of slot ID → bucketed CPM string. +/// Returns a JSON object map of slot ID → bid metadata including the bucketed +/// CPM (`hb_pb`), bidder (`hb_bidder`), and optional ad ID, nurl, and burl. pub(crate) fn build_bid_map( winning_bids: &std::collections::HashMap, granularity: crate::price_bucket::PriceGranularity, -) -> std::collections::HashMap { +) -> serde_json::Map { winning_bids .iter() .filter_map(|(slot_id, bid)| { bid.price.map(|cpm| { let bucket = price_bucket(cpm, granularity); - (slot_id.clone(), bucket) + let mut obj = serde_json::Map::new(); + obj.insert("hb_pb".to_string(), serde_json::Value::String(bucket)); + obj.insert( + "hb_bidder".to_string(), + serde_json::Value::String(bid.bidder.clone()), + ); + if let Some(ref ad_id) = bid.ad_id { + obj.insert( + "hb_adid".to_string(), + serde_json::Value::String(ad_id.clone()), + ); + } + if let Some(ref nurl) = bid.nurl { + obj.insert("nurl".to_string(), serde_json::Value::String(nurl.clone())); + } + if let Some(ref burl) = bid.burl { + obj.insert("burl".to_string(), serde_json::Value::String(burl.clone())); + } + (slot_id.clone(), serde_json::Value::Object(obj)) }) }) .collect() } -/// Build the `__ts_bids` inline script content from a bucketed bid map. -pub(crate) fn build_bids_script(bid_map: &std::collections::HashMap) -> String { - let entries: Vec = bid_map - .iter() - .map(|(slot_id, bucket)| format!("\"{}\":\"{}\"", slot_id, bucket)) - .collect(); - format!("window.__ts_bids={{{}}};", entries.join(",")) +/// Build the `__ts_bids` `` sequences inside the string. +pub(crate) fn build_bids_script(bid_map: &serde_json::Map) -> String { + let json = serde_json::to_string(bid_map).unwrap_or_else(|_| "{}".to_string()); + let escaped = html_escape_for_script(&json); + format!( + "", + escaped + ) } -/// Build the `__ts_ad_slots` inline script content from matched slots. +/// Build the `__ts_ad_slots` `", + escaped + ) } /// Whether the content type requires processing (URL rewriting, HTML injection). From c212ec544138791419b8faed992627101a7a60dc Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 11:47:51 +0530 Subject: [PATCH 019/395] Wire slots_file and orchestrator into adapter; parse creative-opportunities.toml at startup --- Cargo.lock | 8 +++++ .../trusted-server-adapter-fastly/Cargo.toml | 1 + .../trusted-server-adapter-fastly/src/main.rs | 14 ++++++++- crates/trusted-server-core/build.rs | 11 +++---- .../src/creative_opportunities.rs | 30 +++++++++++++------ crates/trusted-server-core/src/lib.rs | 2 +- crates/trusted-server-core/src/settings.rs | 4 ++- 7 files changed, 53 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e06ac75e7..65d1d777c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1151,6 +1151,12 @@ dependencies = [ "wasip2", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "group" version = "0.13.0" @@ -2707,6 +2713,7 @@ dependencies = [ "log-fastly", "serde", "serde_json", + "toml 1.0.7+spec-1.1.0", "trusted-server-core", "urlencoding", ] @@ -2731,6 +2738,7 @@ dependencies = [ "fastly", "flate2", "futures", + "glob", "hex", "hmac", "http", diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index e483ea621..a730efcd6 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -20,6 +20,7 @@ log = { workspace = true } log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +toml = { workspace = true } trusted-server-core = { workspace = true } urlencoding = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 52c869d7f..74414220b 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -39,6 +39,8 @@ use crate::error::to_error_response; use crate::logging::init_logger; use crate::platform::{build_runtime_services, open_kv_store, UnavailableKvStore}; +const CREATIVE_OPPORTUNITIES_TOML: &str = include_str!("../../../creative-opportunities.toml"); + /// Entry point for the Fastly Compute program. /// /// Uses an undecorated `main()` with `Request::from_client()` instead of @@ -80,6 +82,10 @@ fn main() { } }; + let slots_file: trusted_server_core::creative_opportunities::CreativeOpportunitiesFile = + toml::from_str(CREATIVE_OPPORTUNITIES_TOML) + .expect("should parse creative-opportunities.toml"); + let integration_registry = match IntegrationRegistry::new(&settings) { Ok(r) => r, Err(e) => { @@ -103,6 +109,7 @@ fn main() { &orchestrator, &integration_registry, &runtime_services, + &slots_file, req, )) { response.send_to_client(); @@ -114,6 +121,7 @@ async fn route_request( orchestrator: &AuctionOrchestrator, integration_registry: &IntegrationRegistry, runtime_services: &RuntimeServices, + slots_file: &trusted_server_core::creative_opportunities::CreativeOpportunitiesFile, mut req: Request, ) -> Option { // Strip client-spoofable forwarded headers at the edge. @@ -221,8 +229,12 @@ async fn route_request( settings, integration_registry, &publisher_services, + orchestrator, + slots_file, req, - ) { + ) + .await + { Ok(PublisherResponse::Stream { mut response, body, diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index 469c11048..b21cb6845 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -92,14 +92,15 @@ fn main() { let co_path = Path::new(CREATIVE_OPPORTUNITIES_PATH); if co_path.exists() { - let co_content = fs::read_to_string(co_path) - .expect("should read creative-opportunities.toml"); - let co_value: toml::Value = toml::from_str(&co_content) - .expect("creative-opportunities.toml: invalid TOML"); + let co_content = + fs::read_to_string(co_path).expect("should read creative-opportunities.toml"); + let co_value: toml::Value = + toml::from_str(&co_content).expect("creative-opportunities.toml: invalid TOML"); let slot_id_re = regex::Regex::new(r"^[A-Za-z0-9_\-]+$").expect("should compile regex"); if let Some(slots) = co_value.get("slot").and_then(|v| v.as_array()) { for slot in slots { - let id = slot.get("id") + let id = slot + .get("id") .and_then(|v| v.as_str()) .expect("creative-opportunities.toml: slot missing 'id' field"); if !slot_id_re.is_match(id) { diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 7bf3856c2..f051c340f 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -6,9 +6,10 @@ use std::collections::HashMap; -use glob::Pattern; use serde::{Deserialize, Serialize}; +use glob::Pattern; + use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::price_bucket::PriceGranularity; @@ -64,8 +65,9 @@ impl CreativeOpportunitySlot { /// Patterns that cannot be compiled even after normalisation are silently skipped. #[must_use] pub fn matches_path(&self, path: &str) -> bool { - self.page_patterns.iter().any(|pattern| { - match Pattern::new(pattern) { + self.page_patterns + .iter() + .any(|pattern| match Pattern::new(pattern) { Ok(p) => p.matches(path), Err(_) => { let normalised = pattern.replace("**", "*"); @@ -73,8 +75,7 @@ impl CreativeOpportunitySlot { .map(|p| p.matches(path)) .unwrap_or(false) } - } - }) + }) } /// Returns the GAM ad unit path for this slot. @@ -227,7 +228,10 @@ mod tests { #[test] fn glob_matches_article_path() { let slot = make_slot("atf", vec!["/20**"]); - assert!(slot.matches_path("/2024/01/my-article/"), "should match article path"); + assert!( + slot.matches_path("/2024/01/my-article/"), + "should match article path" + ); assert!(!slot.matches_path("/"), "should not match root"); } @@ -243,14 +247,20 @@ mod tests { assert!(validate_slot_id("atf_sidebar_ad").is_ok()); assert!(validate_slot_id("below-content-0").is_ok()); assert!(validate_slot_id("").is_err(), "empty id should fail"); - assert!(validate_slot_id("xss"); + assert!(!inner.contains('<'), "no unescaped < in script content"); + assert!(!inner.contains('>'), "no unescaped > in script content"); + } + + #[test] + fn bid_map_includes_nurl_and_burl() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + make_bid( + "atf_sidebar_ad", + 1.50, + "kargo", + "abc123", + "https://ssp/win", + "https://ssp/bill", + ), + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let entry = map.get("atf_sidebar_ad").expect("should have bid entry"); + let obj = entry.as_object().expect("should be object"); + assert_eq!( + obj.get("hb_pb").and_then(|v| v.as_str()), + Some("1.50"), + "should bucket price with dense granularity" + ); + assert_eq!( + obj.get("hb_bidder").and_then(|v| v.as_str()), + Some("kargo"), + "should include bidder" + ); + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("abc123"), + "should include ad_id" + ); + assert_eq!( + obj.get("nurl").and_then(|v| v.as_str()), + Some("https://ssp/win"), + "should include nurl" + ); + assert_eq!( + obj.get("burl").and_then(|v| v.as_str()), + Some("https://ssp/bill"), + "should include burl" + ); + } + + #[test] + fn bid_map_excludes_slot_when_price_is_none() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "no-price-slot".to_string(), + Bid { + slot_id: "no-price-slot".to_string(), + price: None, + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + assert!( + map.is_empty(), + "slot with no price should be excluded from bid map" + ); + } + + #[test] + fn bids_script_is_xss_safe() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + let script = build_bids_script(&map); + let inner = script + .trim_start_matches(""); + assert!(!inner.contains('<'), "no unescaped < in bids script"); + assert!(!inner.contains('>'), "no unescaped > in bids script"); + } + + #[test] + fn html_escape_encodes_special_chars() { + assert_eq!( + html_escape_for_script("text\\with\\backslash"), + "text\\\\with\\\\backslash", + "should escape backslashes" + ); + assert_eq!( + html_escape_for_script("string\"with\"quotes"), + "string\\\"with\\\"quotes", + "should escape quotes" + ); + assert_eq!( + html_escape_for_script("simple"), + "simple", + "should not change simple text" + ); + assert_eq!( + html_escape_for_script("both\\\"mixed"), + "both\\\\\\\"mixed", + "should escape both backslashes and quotes" + ); + } + } } From b047add10a3f9138949b4ad19e783fca2e3b9a8d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 13:24:00 +0530 Subject: [PATCH 023/395] Enable server-side auction with APS provider and adserver_mock mediator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Enable APS and adserver_mock in auction config; set providers and mediator - Increase auction_timeout_ms from 500ms to 3000ms — 500ms was too tight for HTTPS round-trips to mocktioneer, leaving the mediator zero budget - Fix mediation request: send numeric price instead of opaque encoded_price; mocktioneer requires a decoded price field and does not support encoded_price - Expand creative-opportunities slot page_patterns to include /news/** --- .../src/integrations/adserver_mock.rs | 45 +++++++------------ creative-opportunities.toml | 2 +- trusted-server.toml | 12 ++--- 3 files changed, 22 insertions(+), 37 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 7ed2da595..8ec94a9c5 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -133,36 +133,21 @@ impl AdServerMockProvider { .bids .iter() .map(|bid| { - // Check if this is an APS bid with encoded price (inferred from amznbid in metadata) - let encoded_price = bid - .metadata - .get("amznbid") - .and_then(|v| v.as_str()) - .map(String::from); - - if encoded_price.is_some() { - // APS bid - send encoded price for mediation to decode - json!({ - "imp_id": bid.slot_id, - "encoded_price": encoded_price, - "adm": bid.creative, - "w": bid.width, - "h": bid.height, - "crid": format!("{}-creative", bid.bidder), - "adomain": bid.adomain, - }) - } else { - // Regular bid with decoded price - json!({ - "imp_id": bid.slot_id, - "price": bid.price, - "adm": bid.creative, - "w": bid.width, - "h": bid.height, - "crid": format!("{}-creative", bid.bidder), - "adomain": bid.adomain, - }) - } + // Mocktioneer mediator always requires a numeric `price` field. + // APS bids carry price as an opaque encoded string (`amznbid`) + // that cannot be decoded client-side; use `bid.price` when set + // (a real decoded value) or fall back to a mock floor price for + // test/demo purposes. + let price = bid.price.unwrap_or(1.50); + json!({ + "imp_id": bid.slot_id, + "price": price, + "adm": bid.creative, + "w": bid.width, + "h": bid.height, + "crid": format!("{}-creative", bid.bidder), + "adomain": bid.adomain, + }) }) .collect(); diff --git a/creative-opportunities.toml b/creative-opportunities.toml index b44e215b6..b79d23810 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -5,7 +5,7 @@ id = "atf_sidebar_ad" gam_unit_path = "/21765378893/publisher/atf-sidebar" div_id = "div-atf-sidebar" -page_patterns = ["/20**"] +page_patterns = ["/", "/20**", "/news/**"] formats = [{ width = 300, height = 250 }] floor_price = 0.50 diff --git a/trusted-server.toml b/trusted-server.toml index c2ecab335..8036b7ec4 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -161,16 +161,16 @@ rewrite_script = true [auction] enabled = true -providers = ["prebid"] -# mediator = "adserver_mock" # will use mediator when set +providers = ["prebid", "aps"] +mediator = "adserver_mock" timeout_ms = 2000 # Context keys the JS client is allowed to forward into auction requests. # Keys not in this list are silently dropped. An empty list blocks all keys. allowed_context_keys = ["permutive_segments"] [integrations.aps] -enabled = false -pub_id = "your-aps-publisher-id" +enabled = true +pub_id = "test-pub" endpoint = "https://origin-mocktioneer.cdintel.com/e/dtb/bid" timeout_ms = 1000 @@ -180,7 +180,7 @@ container_id = "GTM-XXXXXX" # upstream_url = "https://www.googletagmanager.com" [integrations.adserver_mock] -enabled = false +enabled = true endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" timeout_ms = 1000 @@ -192,6 +192,6 @@ permutive_segments = "permutive" [creative_opportunities] gam_network_id = "21765378893" -auction_timeout_ms = 500 +auction_timeout_ms = 3000 price_granularity = "dense" From 6a5df1060471818c8335178ce35ecd88978aa2ac Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 13:35:24 +0530 Subject: [PATCH 024/395] Fix adserver_mock test for numeric price; fix GPT JS formatting --- .../js/lib/src/integrations/gpt/index.test.ts | 150 +++++++++--------- crates/js/lib/src/integrations/gpt/index.ts | 14 +- .../src/integrations/adserver_mock.rs | 19 +-- 3 files changed, 91 insertions(+), 92 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/src/integrations/gpt/index.test.ts index 0a6993818..7e2783f2f 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/src/integrations/gpt/index.test.ts @@ -1,20 +1,20 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest' +import { describe, it, expect, vi, beforeEach } from 'vitest'; describe('installTsAdInit', () => { beforeEach(() => { - vi.resetModules() - delete (window as any).__ts_ad_slots - delete (window as any).__ts_bids - delete (window as any).__tsAdInit + vi.resetModules(); + delete (window as any).__ts_ad_slots; + delete (window as any).__ts_bids; + delete (window as any).__tsAdInit; // jsdom does not implement navigator.sendBeacon; polyfill it for tests if (!('sendBeacon' in navigator)) { Object.defineProperty(navigator, 'sendBeacon', { value: vi.fn().mockReturnValue(true), writable: true, configurable: true, - }) + }); } - }) + }); it('reads window.__ts_bids synchronously and applies bid targeting before refresh', async () => { const mockSlot = { @@ -22,19 +22,19 @@ describe('installTsAdInit', () => { setTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('atf'), getTargeting: vi.fn().mockReturnValue(['abc']), - } + }; const mockPubads = { enableSingleRequest: vi.fn(), addEventListener: vi.fn(), refresh: vi.fn(), - } - ;(window as any).googletag = { + }; + (window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlot), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), - } - ;(window as any).__ts_ad_slots = [ + }; + (window as any).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -42,8 +42,8 @@ describe('installTsAdInit', () => { formats: [[300, 250]], targeting: { pos: 'atf' }, }, - ] - ;(window as any).__ts_bids = { + ]; + (window as any).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -51,47 +51,47 @@ describe('installTsAdInit', () => { nurl: 'https://ssp/win', burl: 'https://ssp/bill', }, - } + }; - const fetchSpy = vi.spyOn(global, 'fetch') + const fetchSpy = vi.spyOn(global, 'fetch'); - const { installTsAdInit } = await import('./index') - installTsAdInit() - ;(window as any).__tsAdInit() + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as any).__tsAdInit(); - expect(fetchSpy).not.toHaveBeenCalled() - expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00') - expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'kargo') - expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1') - expect(mockPubads.refresh).toHaveBeenCalled() + expect(fetchSpy).not.toHaveBeenCalled(); + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00'); + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'kargo'); + expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); + expect(mockPubads.refresh).toHaveBeenCalled(); - fetchSpy.mockRestore() - }) + fetchSpy.mockRestore(); + }); it('fires both nurl and burl via sendBeacon on slotRenderEnded when our bid won', async () => { - const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) - let capturedListener: ((e: any) => void) | undefined + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let capturedListener: ((e: any) => void) | undefined; const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('atf'), getTargeting: vi.fn().mockReturnValue(['abc']), - } + }; const mockPubads = { enableSingleRequest: vi.fn(), refresh: vi.fn(), addEventListener: vi.fn((event: string, fn: (e: any) => void) => { - if (event === 'slotRenderEnded') capturedListener = fn + if (event === 'slotRenderEnded') capturedListener = fn; }), - } - ;(window as any).googletag = { + }; + (window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlot), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), - } - ;(window as any).__ts_ad_slots = [ + }; + (window as any).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -99,8 +99,8 @@ describe('installTsAdInit', () => { formats: [[300, 250]], targeting: {}, }, - ] - ;(window as any).__ts_bids = { + ]; + (window as any).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -108,44 +108,44 @@ describe('installTsAdInit', () => { nurl: 'https://ssp/win', burl: 'https://ssp/bill', }, - } + }; - const { installTsAdInit } = await import('./index') - installTsAdInit() - ;(window as any).__tsAdInit() + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as any).__tsAdInit(); - expect(capturedListener).toBeDefined() - capturedListener!({ isEmpty: false, slot: mockSlot }) + expect(capturedListener).toBeDefined(); + capturedListener!({ isEmpty: false, slot: mockSlot }); - expect(beaconSpy).toHaveBeenCalledWith('https://ssp/win') - expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill') - beaconSpy.mockRestore() - }) + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/win'); + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill'); + beaconSpy.mockRestore(); + }); it('does not fire nurl/burl when bid did not win GAM line item', async () => { - const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) - let capturedListener: ((e: any) => void) | undefined + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let capturedListener: ((e: any) => void) | undefined; const mockSlotNoMatch = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('atf'), getTargeting: vi.fn().mockReturnValue(['OTHER_BID_ID']), - } + }; const mockPubads = { enableSingleRequest: vi.fn(), refresh: vi.fn(), addEventListener: vi.fn((event: string, fn: (e: any) => void) => { - if (event === 'slotRenderEnded') capturedListener = fn + if (event === 'slotRenderEnded') capturedListener = fn; }), - } - ;(window as any).googletag = { + }; + (window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlotNoMatch), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), - } - ;(window as any).__ts_ad_slots = [ + }; + (window as any).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -153,8 +153,8 @@ describe('installTsAdInit', () => { formats: [[300, 250]], targeting: {}, }, - ] - ;(window as any).__ts_bids = { + ]; + (window as any).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -162,24 +162,24 @@ describe('installTsAdInit', () => { nurl: 'https://ssp/win', burl: 'https://ssp/bill', }, - } + }; - const { installTsAdInit } = await import('./index') - installTsAdInit() - ;(window as any).__tsAdInit() - capturedListener!({ isEmpty: false, slot: mockSlotNoMatch }) + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as any).__tsAdInit(); + capturedListener!({ isEmpty: false, slot: mockSlotNoMatch }); - expect(beaconSpy).not.toHaveBeenCalled() - beaconSpy.mockRestore() - }) + expect(beaconSpy).not.toHaveBeenCalled(); + beaconSpy.mockRestore(); + }); it('calls refresh even when __ts_bids is empty (graceful fallback)', async () => { const mockPubads = { enableSingleRequest: vi.fn(), addEventListener: vi.fn(), refresh: vi.fn(), - } - ;(window as any).googletag = { + }; + (window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue({ addService: vi.fn().mockReturnThis(), @@ -187,14 +187,14 @@ describe('installTsAdInit', () => { }), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), - } - ;(window as any).__ts_ad_slots = [] - ;(window as any).__ts_bids = {} + }; + (window as any).__ts_ad_slots = []; + (window as any).__ts_bids = {}; - const { installTsAdInit } = await import('./index') - installTsAdInit() - ;(window as any).__tsAdInit() + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as any).__tsAdInit(); - expect(mockPubads.refresh).toHaveBeenCalled() - }) -}) + expect(mockPubads.refresh).toHaveBeenCalled(); + }); +}); diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 1494d793f..95b6d4279 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -217,7 +217,11 @@ export function installTsAdInit(): void { g.cmd?.push(() => { slots .map((slot) => { - const gptSlot = g.defineSlot?.(slot.gam_unit_path, slot.formats as Array, slot.div_id); + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats as Array, + slot.div_id + ); if (!gptSlot) return null; gptSlot.addService(g.pubads!()); Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v)); @@ -280,13 +284,13 @@ export function installSlimPrebidLoader(): void { // regardless of script order, the module also checks for a pre-set enable flag // immediately after registering the function. if (typeof window !== 'undefined') { - const win = window as Record + const win = window as Record; - win.__tsjs_installGptShim = installGptShim + win.__tsjs_installGptShim = installGptShim; if (win.__tsjs_gpt_enabled === true) { - installGptShim() + installGptShim(); } - installTsAdInit() + installTsAdInit(); } diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 8ec94a9c5..3a42ec2a0 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -675,20 +675,15 @@ mod tests { let bid = &bidder_resp["bids"][0]; assert_eq!(bid["imp_id"], "slot-1"); - // Key assertions for APS-style encoded price bids: - // 1. Should NOT have "price" field (or it should be null) - assert!( - bid["price"].is_null(), - "APS bids should not have decoded price, got: {:?}", - bid["price"] - ); - // 2. Should have "encoded_price" field + // APS bids have no decoded price (bid.price == None), so the mock floor + // price (1.50) is used. Mocktioneer requires a numeric price field and + // does not accept an opaque encoded_price string. assert_eq!( - bid["encoded_price"].as_str(), - Some("encoded-price-value"), - "APS bids should have encoded_price from metadata" + bid["price"].as_f64(), + Some(1.50), + "APS bids with no decoded price should fall back to mock floor price 1.50" ); - // 3. adm should be null (not a string) + // adm should be null (not a string) assert!( bid["adm"].is_null(), "Creative-less bids should have null adm, got: {:?}", From e6c18ad5ec4de17713a840d2c44e0d2b532b5946 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 14:13:49 +0530 Subject: [PATCH 025/395] Replace explicit any in GPT integration with typed interfaces Define SlotRenderEndedEvent, SlotRenderEvent, and TestWindow types to eliminate all @typescript-eslint/no-explicit-any violations in gpt/index.ts and gpt/index.test.ts. Extend GptWindow with __tsjs_slim_prebid_url so installSlimPrebidLoader avoids the any cast. --- .../js/lib/src/integrations/gpt/index.test.ts | 61 ++++++++++++------- crates/js/lib/src/integrations/gpt/index.ts | 15 +++-- 2 files changed, 49 insertions(+), 27 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/src/integrations/gpt/index.test.ts index 7e2783f2f..e908a201e 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/src/integrations/gpt/index.test.ts @@ -1,11 +1,26 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; +interface SlotRenderEvent { + isEmpty: boolean; + slot: { + getSlotElementId(): string; + getTargeting(key: string): string[]; + }; +} + +type TestWindow = Window & { + googletag?: unknown; + __ts_ad_slots?: unknown; + __ts_bids?: unknown; + __tsAdInit?: () => void; +}; + describe('installTsAdInit', () => { beforeEach(() => { vi.resetModules(); - delete (window as any).__ts_ad_slots; - delete (window as any).__ts_bids; - delete (window as any).__tsAdInit; + delete (window as TestWindow).__ts_ad_slots; + delete (window as TestWindow).__ts_bids; + delete (window as TestWindow).__tsAdInit; // jsdom does not implement navigator.sendBeacon; polyfill it for tests if (!('sendBeacon' in navigator)) { Object.defineProperty(navigator, 'sendBeacon', { @@ -28,13 +43,13 @@ describe('installTsAdInit', () => { addEventListener: vi.fn(), refresh: vi.fn(), }; - (window as any).googletag = { + (window as TestWindow).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlot), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), }; - (window as any).__ts_ad_slots = [ + (window as TestWindow).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -43,7 +58,7 @@ describe('installTsAdInit', () => { targeting: { pos: 'atf' }, }, ]; - (window as any).__ts_bids = { + (window as TestWindow).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -57,7 +72,7 @@ describe('installTsAdInit', () => { const { installTsAdInit } = await import('./index'); installTsAdInit(); - (window as any).__tsAdInit(); + (window as TestWindow).__tsAdInit!(); expect(fetchSpy).not.toHaveBeenCalled(); expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00'); @@ -70,7 +85,7 @@ describe('installTsAdInit', () => { it('fires both nurl and burl via sendBeacon on slotRenderEnded when our bid won', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); - let capturedListener: ((e: any) => void) | undefined; + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; const mockSlot = { addService: vi.fn().mockReturnThis(), @@ -81,17 +96,17 @@ describe('installTsAdInit', () => { const mockPubads = { enableSingleRequest: vi.fn(), refresh: vi.fn(), - addEventListener: vi.fn((event: string, fn: (e: any) => void) => { + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { if (event === 'slotRenderEnded') capturedListener = fn; }), }; - (window as any).googletag = { + (window as TestWindow).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlot), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), }; - (window as any).__ts_ad_slots = [ + (window as TestWindow).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -100,7 +115,7 @@ describe('installTsAdInit', () => { targeting: {}, }, ]; - (window as any).__ts_bids = { + (window as TestWindow).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -112,7 +127,7 @@ describe('installTsAdInit', () => { const { installTsAdInit } = await import('./index'); installTsAdInit(); - (window as any).__tsAdInit(); + (window as TestWindow).__tsAdInit!(); expect(capturedListener).toBeDefined(); capturedListener!({ isEmpty: false, slot: mockSlot }); @@ -124,7 +139,7 @@ describe('installTsAdInit', () => { it('does not fire nurl/burl when bid did not win GAM line item', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); - let capturedListener: ((e: any) => void) | undefined; + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; const mockSlotNoMatch = { addService: vi.fn().mockReturnThis(), @@ -135,17 +150,17 @@ describe('installTsAdInit', () => { const mockPubads = { enableSingleRequest: vi.fn(), refresh: vi.fn(), - addEventListener: vi.fn((event: string, fn: (e: any) => void) => { + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { if (event === 'slotRenderEnded') capturedListener = fn; }), }; - (window as any).googletag = { + (window as TestWindow).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlotNoMatch), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), }; - (window as any).__ts_ad_slots = [ + (window as TestWindow).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -154,7 +169,7 @@ describe('installTsAdInit', () => { targeting: {}, }, ]; - (window as any).__ts_bids = { + (window as TestWindow).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -166,7 +181,7 @@ describe('installTsAdInit', () => { const { installTsAdInit } = await import('./index'); installTsAdInit(); - (window as any).__tsAdInit(); + (window as TestWindow).__tsAdInit!(); capturedListener!({ isEmpty: false, slot: mockSlotNoMatch }); expect(beaconSpy).not.toHaveBeenCalled(); @@ -179,7 +194,7 @@ describe('installTsAdInit', () => { addEventListener: vi.fn(), refresh: vi.fn(), }; - (window as any).googletag = { + (window as TestWindow).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue({ addService: vi.fn().mockReturnThis(), @@ -188,12 +203,12 @@ describe('installTsAdInit', () => { pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), }; - (window as any).__ts_ad_slots = []; - (window as any).__ts_bids = {}; + (window as TestWindow).__ts_ad_slots = []; + (window as TestWindow).__ts_bids = {}; const { installTsAdInit } = await import('./index'); installTsAdInit(); - (window as any).__tsAdInit(); + (window as TestWindow).__tsAdInit!(); expect(mockPubads.refresh).toHaveBeenCalled(); }); diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 95b6d4279..ffb4a687f 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -32,13 +32,19 @@ interface GoogleTagSlot { getSlotElementId(): string; setTargeting(key: string, value: string | string[]): GoogleTagSlot; addService(service: GoogleTagPubAdsService): GoogleTagSlot; + getTargeting?(key: string): string[]; +} + +interface SlotRenderEndedEvent { + isEmpty: boolean; + slot: GoogleTagSlot; } interface GoogleTagPubAdsService { setTargeting(key: string, value: string | string[]): GoogleTagPubAdsService; getTargeting(key: string): string[]; enableSingleRequest(): void; - addEventListener(event: string, fn: (e: any) => void): void; + addEventListener(event: string, fn: (e: SlotRenderEndedEvent) => void): void; refresh(): void; } @@ -57,6 +63,7 @@ interface GoogleTag { type GptWindow = Window & { googletag?: Partial; + __tsjs_slim_prebid_url?: string; }; // ------------------------------------------------------------------ @@ -237,7 +244,7 @@ export function installTsAdInit(): void { g.pubads!().enableSingleRequest(); g.enableServices?.(); - g.pubads!().addEventListener?.('slotRenderEnded', (event: any) => { + g.pubads!().addEventListener?.('slotRenderEnded', (event: SlotRenderEndedEvent) => { const slotId: string = event.slot?.getSlotElementId?.() ?? ''; const bid = bids[slotId] ?? {}; const ourBidWon = @@ -265,7 +272,7 @@ export function installTsAdInit(): void { * the slim-Prebid bundle build target ships in a later phase). */ export function installSlimPrebidLoader(): void { - const url = (window as any).__tsjs_slim_prebid_url as string | undefined; + const url = (window as GptWindow).__tsjs_slim_prebid_url; if (!url) return; window.addEventListener('load', () => { const script = document.createElement('script'); @@ -284,7 +291,7 @@ export function installSlimPrebidLoader(): void { // regardless of script order, the module also checks for a pre-set enable flag // immediately after registering the function. if (typeof window !== 'undefined') { - const win = window as Record; + const win = window as unknown as Record; win.__tsjs_installGptShim = installGptShim; From 74bbc25b4b52ab1b5ea012894d109c67e606ceb2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 15:33:39 +0530 Subject: [PATCH 026/395] Update creative-opportunities config to real autoblog.com GAM values Set gam_network_id to 88059007 (autoblog production network). Update atf_sidebar_ad slot to /88059007/autoblog/news with div_id ad-atf_sidebar-0-_r_2_ (desktop ATF sidebar, 300x250); restrict page_patterns to article paths only (/20**, /news/**) since that div does not exist on the homepage. Add homepage_header_ad slot targeting /88059007/autoblog/homepage with ad-header-0-_R_jpalubtak5lb_ for 970x90/728x90/970x250 leaderboard formats. Reduce auction_timeout_ms from 3000 to 500 to cap TTFB at the spec-recommended ceiling. --- creative-opportunities.toml | 21 ++++++++++++++++++--- trusted-server.toml | 4 ++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/creative-opportunities.toml b/creative-opportunities.toml index b79d23810..0261110a2 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -3,9 +3,9 @@ [[slot]] id = "atf_sidebar_ad" -gam_unit_path = "/21765378893/publisher/atf-sidebar" -div_id = "div-atf-sidebar" -page_patterns = ["/", "/20**", "/news/**"] +gam_unit_path = "/88059007/autoblog/news" +div_id = "ad-atf_sidebar-0-_r_2_" +page_patterns = ["/20**", "/news/**"] formats = [{ width = 300, height = 250 }] floor_price = 0.50 @@ -15,3 +15,18 @@ zone = "atfSidebar" [slot.providers.aps] slot_id = "aps-slot-atf-sidebar" + +[[slot]] +id = "homepage_header_ad" +gam_unit_path = "/88059007/autoblog/homepage" +div_id = "ad-header-0-_R_jpalubtak5lb_" +page_patterns = ["/"] +formats = [{ width = 970, height = 90 }, { width = 728, height = 90 }, { width = 970, height = 250 }] +floor_price = 0.50 + +[slot.targeting] +pos = "atf" +zone = "header" + +[slot.providers.aps] +slot_id = "aps-slot-homepage-header" diff --git a/trusted-server.toml b/trusted-server.toml index 8036b7ec4..da00c3ed7 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -191,7 +191,7 @@ timeout_ms = 1000 permutive_segments = "permutive" [creative_opportunities] -gam_network_id = "21765378893" -auction_timeout_ms = 3000 +gam_network_id = "88059007" +auction_timeout_ms = 500 price_granularity = "dense" From 51aba8f1b48a5a2c18bf1fb3df5ed76fc66d837c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 17:49:49 +0530 Subject: [PATCH 027/395] Update auction timeout and APS slot ID bug --- .../src/integrations/aps.rs | 139 ++++++++++++++++-- trusted-server.toml | 2 +- 2 files changed, 128 insertions(+), 13 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 79eca5a32..ba6c14bbd 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -286,24 +286,46 @@ impl IntegrationConfig for ApsConfig { /// Amazon APS auction provider. pub struct ApsAuctionProvider { config: ApsConfig, + // Maps APS slot ID → creative opportunity slot ID for the in-flight request. + // Written by request_bids before the async send; read by parse_response when the + // response arrives. Safe because Fastly Compute runs each request in an isolated + // single-threaded Wasm instance — the Mutex never contends in practice. + slot_id_map: std::sync::Mutex>, } impl ApsAuctionProvider { /// Create a new APS auction provider. #[must_use] pub fn new(config: ApsConfig) -> Self { - Self { config } + Self { + config, + slot_id_map: std::sync::Mutex::new(HashMap::new()), + } } /// Convert unified `AuctionRequest` to APS TAM bid request format. /// + /// Returns the serialisable `ApsBidRequest` and a map of APS slot ID → + /// creative-opportunity slot ID so the caller can remap bids in the response. /// Populates consent fields (GDPR, US Privacy, GPP) from the /// [`ConsentContext`](crate::consent::ConsentContext) attached to the request. - fn to_aps_request(&self, request: &AuctionRequest) -> ApsBidRequest { + fn to_aps_request(&self, request: &AuctionRequest) -> (ApsBidRequest, HashMap) { + let mut slot_id_map: HashMap = HashMap::new(); let slots: Vec = request .slots .iter() .map(|slot| { + // Use the APS-specific slot ID from [slot.providers.aps] if configured; + // fall back to the creative-opportunity slot ID otherwise. + let aps_slot_id = slot + .bidders + .get("aps") + .and_then(|p| p.get("slotID")) + .and_then(|v| v.as_str()) + .unwrap_or(&slot.id) + .to_string(); + slot_id_map.insert(aps_slot_id.clone(), slot.id.clone()); + // Extract sizes from banner formats let sizes: Vec<[u32; 2]> = slot .formats @@ -313,7 +335,7 @@ impl ApsAuctionProvider { .collect(); ApsSlot { - slot_id: slot.id.clone(), + slot_id: aps_slot_id, sizes, slot_name: Some(slot.id.clone()), } @@ -337,7 +359,7 @@ impl ApsAuctionProvider { }) }); - ApsBidRequest { + let bid_request = ApsBidRequest { pub_id: self.config.pub_id.clone(), slots, page_url: request.publisher.page_url.clone(), @@ -347,7 +369,8 @@ impl ApsAuctionProvider { us_privacy, gpp, gpp_sid, - } + }; + (bid_request, slot_id_map) } /// Parse size string (e.g., "300x250") into width and height. @@ -433,9 +456,19 @@ impl ApsAuctionProvider { aps_response.contextual.slots.len() ); + let slot_map = self + .slot_id_map + .lock() + .expect("should lock APS slot id map"); for slot in aps_response.contextual.slots { match self.parse_aps_slot(&slot) { - Ok(bid) => { + Ok(mut bid) => { + // Remap APS slot ID (e.g. "aps-slot-atf-sidebar") back to the + // creative-opportunity slot ID (e.g. "atf_sidebar_ad") so the + // mediator and bid_map can match by creative slot ID. + if let Some(creative_id) = slot_map.get(&bid.slot_id) { + bid.slot_id = creative_id.clone(); + } let encoded_price = bid .metadata .get("amznbid") @@ -485,8 +518,13 @@ impl AuctionProvider for ApsAuctionProvider { self.config.pub_id ); - // Transform to APS format - let aps_request = self.to_aps_request(request); + // Transform to APS format; store the APS-slot-ID → creative-slot-ID map so + // parse_response can remap bids back to the creative opportunity slot ID. + let (aps_request, slot_id_map) = self.to_aps_request(request); + *self + .slot_id_map + .lock() + .expect("should lock APS slot id map") = slot_id_map; // Serialize to JSON let aps_json = @@ -703,7 +741,7 @@ mod tests { let provider = ApsAuctionProvider::new(config); let auction_request = create_test_auction_request(); - let aps_request = provider.to_aps_request(&auction_request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&auction_request); // Verify basic fields assert_eq!(aps_request.pub_id, "5128"); @@ -729,6 +767,83 @@ mod tests { assert_eq!(slot2.sizes[0], [300, 250]); } + #[test] + fn aps_slot_id_from_bidders_map_used_in_request_and_remapped_in_response() { + use serde_json::json; + + let config = ApsConfig { + enabled: true, + pub_id: "5128".to_string(), + endpoint: default_endpoint(), + timeout_ms: 800, + }; + let provider = ApsAuctionProvider::new(config); + + let mut bidders = HashMap::new(); + bidders.insert( + "aps".to_string(), + json!({ "slotID": "aps-slot-atf-sidebar" }), + ); + let request = AuctionRequest { + id: "test".to_string(), + slots: vec![AdSlot { + id: "atf_sidebar_ad".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: None, + targeting: HashMap::new(), + bidders, + }], + publisher: PublisherInfo { + domain: "example.com".to_string(), + page_url: None, + }, + user: UserInfo { + id: "user-1".to_string(), + fresh_id: "fresh-1".to_string(), + consent: None, + }, + device: None, + site: None, + context: HashMap::new(), + }; + + let (aps_request, slot_id_map) = provider.to_aps_request(&request); + assert_eq!( + aps_request.slots[0].slot_id, "aps-slot-atf-sidebar", + "should send configured APS slot ID to APS" + ); + assert_eq!( + slot_id_map.get("aps-slot-atf-sidebar").map(String::as_str), + Some("atf_sidebar_ad"), + "should build reverse map from APS slot ID to creative slot ID" + ); + + *provider.slot_id_map.lock().expect("should lock") = slot_id_map; + + let aps_response = json!({ + "contextual": { + "slots": [{ + "slotID": "aps-slot-atf-sidebar", + "size": "300x250", + "fif": "1", + "amznbid": "1gtm3q", + "meta": ["slotID"] + }] + } + }); + + let response = provider.parse_aps_response(&aps_response, 100); + assert_eq!(response.bids.len(), 1, "should parse one bid"); + assert_eq!( + response.bids[0].slot_id, "atf_sidebar_ad", + "bid slot_id should be remapped to creative slot ID" + ); + } + #[test] fn test_aps_response_parsing_success() { let config = ApsConfig { @@ -957,7 +1072,7 @@ mod tests { ..Default::default() }); - let aps_request = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request); // Verify GDPR consent let gdpr = aps_request.gdpr.expect("should have gdpr"); @@ -986,7 +1101,7 @@ mod tests { let provider = ApsAuctionProvider::new(config); let request = create_test_auction_request(); // consent is None - let aps_request = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request); assert!(aps_request.gdpr.is_none()); assert!(aps_request.us_privacy.is_none()); @@ -1013,7 +1128,7 @@ mod tests { ..Default::default() }); - let aps_request = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request); let json = serde_json::to_value(&aps_request).expect("should serialize"); // GDPR fields present diff --git a/trusted-server.toml b/trusted-server.toml index da00c3ed7..43e090fea 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -192,6 +192,6 @@ permutive_segments = "permutive" [creative_opportunities] gam_network_id = "88059007" -auction_timeout_ms = 500 +auction_timeout_ms = 1500 price_granularity = "dense" From 3d51fe487e68d08621b0c6a5ffa1364406f45ac1 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 18:43:18 +0530 Subject: [PATCH 028/395] Call __tsAdInit after injecting __ts_bids into page The bids script set window.__ts_bids but never invoked the __tsAdInit function, leaving GPT slots undefined and server-side targeting (hb_pb, hb_bidder) never applied. Both the winning-bid path (build_bids_script) and the no-auction fallback (html_processor None branch) now guard-call the function after the assignment. --- crates/trusted-server-core/src/html_processor.rs | 2 +- crates/trusted-server-core/src/publisher.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 9ef6edb68..45e066609 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -301,7 +301,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let script_guard = state.read().expect("should read bid state"); let bids_script = match &*script_guard { Some(s) => s.clone(), - None => r#""# + None => r#""# .to_string(), }; end_tag.before(&bids_script, ContentType::Html); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 73e489dc6..193f702c3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -883,7 +883,7 @@ pub(crate) fn build_bids_script(bid_map: &serde_json::Mapwindow.__ts_bids=JSON.parse(\"{}\");", + "", escaped ) } From 4cf6d98c3adae70c1fdec3ca1c97f531136a16ef Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 18:50:33 +0530 Subject: [PATCH 029/395] Fix format error --- crates/trusted-server-core/src/html_processor.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 45e066609..a3608d9ec 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -296,8 +296,8 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso move |el| { let state = state.clone(); if let Some(handlers) = el.end_tag_handlers() { - let handler: EndTagHandler<'static> = - Box::new(move |end_tag: &mut EndTag<'_>| { + let handler: EndTagHandler<'static> = Box::new( + move |end_tag: &mut EndTag<'_>| { let script_guard = state.read().expect("should read bid state"); let bids_script = match &*script_guard { Some(s) => s.clone(), @@ -306,7 +306,8 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso }; end_tag.before(&bids_script, ContentType::Html); Ok(()) - }); + }, + ); handlers.push(handler); } Ok(()) From e06af4b0fddee2f6e1ecffba436a7af4f333f247 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 19:36:41 +0530 Subject: [PATCH 030/395] Add PBS inline bidder params via creative-opportunities.toml Adds [slot.providers.pbs.bidders] support so PBS bidder params live in creative-opportunities.toml alongside APS params, without needing PBS stored requests configured server-side. PrebidAuctionProvider now sends imp.ext.prebid.storedrequest.id as a fallback for slots with no inline PBS params, and skips non-PBS provider keys (e.g. "aps") that belong to separate auction providers. PrebidImpExt gains an optional storedrequest field; empty bidder maps are omitted during serialisation. Wires mocktioneer and criteo (placeholder IDs) for both autoblog creative-opportunity slots. --- .../src/creative_opportunities.rs | 66 +++++++++- .../src/integrations/prebid.rs | 116 ++++++++++++++++-- crates/trusted-server-core/src/openrtb.rs | 14 ++- creative-opportunities.toml | 8 ++ 4 files changed, 191 insertions(+), 13 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index f051c340f..a7fd99cb6 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -99,7 +99,8 @@ impl CreativeOpportunitySlot { /// Converts this slot into an [`AdSlot`] ready for use in an auction request. /// - /// Provider-specific params (e.g., APS `slotID`) are wired into the `bidders` map. + /// Provider-specific params (e.g., APS `slotID`, PBS bidder params) are wired + /// into the `bidders` map keyed by provider/bidder name. #[must_use] pub fn to_ad_slot(&self, gam_network_id: &str) -> AdSlot { let _ = gam_network_id; @@ -110,6 +111,11 @@ impl CreativeOpportunitySlot { serde_json::json!({ "slotID": aps.slot_id }), ); } + if let Some(ref pbs) = self.providers.pbs { + for (bidder_name, params) in &pbs.bidders { + bidders.insert(bidder_name.clone(), params.clone()); + } + } AdSlot { id: self.id.clone(), formats: self @@ -155,6 +161,8 @@ impl CreativeOpportunityFormat { pub struct SlotProviders { /// Amazon Publisher Services (APS/TAM) slot parameters. pub aps: Option, + /// Prebid Server (PBS) slot parameters. + pub pbs: Option, } /// APS-specific parameters for a slot. @@ -164,6 +172,24 @@ pub struct ApsSlotParams { pub slot_id: String, } +/// PBS-specific parameters for a slot. +/// +/// Bidder params are sent inline to Prebid Server so bidder credentials +/// stay in `creative-opportunities.toml` rather than in PBS stored requests. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct PbsSlotParams { + /// Per-bidder params keyed by bidder name (must match PBS adapter name). + /// + /// Example in TOML: + /// ```toml + /// [slot.providers.pbs.bidders] + /// mocktioneer = { bid = 2.00 } + /// criteo = { networkId = 123456, pubid = "123456" } + /// ``` + #[serde(default)] + pub bidders: HashMap, +} + /// TOML file structure for creative opportunity slot definitions. #[derive(Debug, Clone, Deserialize, Default)] pub struct CreativeOpportunitiesFile { @@ -293,6 +319,44 @@ mod tests { ); } + #[test] + fn to_ad_slot_wires_pbs_bidder_params_into_bidders() { + let mut slot = make_slot("atf_sidebar_ad", vec!["/"]); + slot.providers.pbs = Some(PbsSlotParams { + bidders: [ + ( + "mocktioneer".to_string(), + serde_json::json!({ "bid": 2.00 }), + ), + ( + "criteo".to_string(), + serde_json::json!({ "networkId": 123456, "pubid": "123456" }), + ), + ] + .into_iter() + .collect(), + }); + let ad_slot = slot.to_ad_slot("88059007"); + let mock_params = ad_slot + .bidders + .get("mocktioneer") + .expect("should have mocktioneer bidder"); + assert_eq!( + mock_params.get("bid").and_then(|v| v.as_f64()), + Some(2.0), + "should wire mocktioneer bid param" + ); + let criteo_params = ad_slot + .bidders + .get("criteo") + .expect("should have criteo bidder"); + assert_eq!( + criteo_params.get("networkId").and_then(|v| v.as_i64()), + Some(112141), + "should wire criteo networkId param" + ); + } + #[test] fn to_ad_slot_sets_floor_price_and_formats() { let slot = make_slot("atf", vec!["/"]); diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 62e112c77..46b87cc0e 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -26,8 +26,8 @@ use crate::integrations::{ }; use crate::openrtb::{ to_openrtb_i32, Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, ImpExt, - OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, RequestExt, Site, ToExt, - TrustedServerExt, User, UserExt, + ImpStoredRequest, OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, + RequestExt, Site, ToExt, TrustedServerExt, User, UserExt, }; use crate::platform::RuntimeServices; use crate::request_signing::{RequestSigner, SigningParams, SIGNING_VERSION}; @@ -529,22 +529,27 @@ impl PrebidAuctionProvider { // Build the bidder map for PBS. // The JS adapter sends "trustedServer" as the bidder (our orchestrator // adapter name). Replace it with the real PBS bidders from config. - // Pass through any other bidders with their params as-is. + // Only pass through keys that are known PBS bidders — skip provider-specific + // keys like "aps" which belong to their own separate auction provider. let mut bidder: HashMap = HashMap::new(); for (name, params) in &slot.bidders { if name == TRUSTED_SERVER_BIDDER { bidder.extend(expand_trusted_server_bidders(&self.config.bidders, params)); - } else { + } else if self.config.bidders.iter().any(|b| b == name) { bidder.insert(name.clone(), params.clone()); } } - // Fallback to config bidders if none provided - if bidder.is_empty() { - for b in &self.config.bidders { - bidder.insert(b.clone(), Json::Object(serde_json::Map::new())); - } - } + // When no inline PBS bidder params exist (e.g. creative-opportunity slots + // whose PBS params live in stored requests), tell PBS to resolve bidder + // config from the stored request keyed by this slot ID. + let storedrequest = if bidder.is_empty() { + Some(ImpStoredRequest { + id: slot.id.clone(), + }) + } else { + None + }; // Apply zone-specific bid param overrides when configured. for (name, params) in &mut bidder { @@ -582,7 +587,10 @@ impl PrebidAuctionProvider { secure: Some(true), // require HTTPS creatives tagid: Some(slot.id.clone()), ext: ImpExt { - prebid: PrebidImpExt { bidder }, + prebid: PrebidImpExt { + bidder, + storedrequest, + }, } .to_ext(), ..Default::default() @@ -3044,4 +3052,90 @@ fixed_bottom = {placementId = "_s2sBottom"} assert_eq!(statuses[0]["bidder"], "kargo"); assert_eq!(statuses[1]["status"], "timeout"); } + + // ======================================================================== + // PBS stored request tests + // ======================================================================== + + #[test] + fn to_openrtb_uses_stored_request_when_slot_has_no_pbs_bidder_params() { + // Slot only has "aps" provider — not a PBS bidder + let slot = make_slot( + "atf_sidebar_ad", + HashMap::from([("aps".to_string(), json!({"slotID": "aps-slot-atf-sidebar"}))]), + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(base_config(), &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("bidder").is_none(), + "should not send inline bidder params when using stored request" + ); + assert_eq!( + prebid["storedrequest"]["id"], "atf_sidebar_ad", + "should use slot id as stored request id" + ); + } + + #[test] + fn to_openrtb_uses_stored_request_when_slot_has_empty_bidders() { + let slot = make_slot("homepage_header_ad", HashMap::new()); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(base_config(), &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert_eq!( + prebid["storedrequest"]["id"], "homepage_header_ad", + "should use slot id as stored request id for slot with no bidder map" + ); + } + + #[test] + fn to_openrtb_uses_inline_bidder_params_not_stored_request_for_trusted_server_slots() { + let mut config = base_config(); + config.bidders = vec!["kargo".to_string()]; + + let slot = make_ts_slot( + "in_content_ad", + &json!({ "kargo": { "placementId": "client_123" } }), + None, + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(config, &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("storedrequest").is_none(), + "should not use stored request when inline bidder params are present" + ); + assert_eq!( + prebid["bidder"]["kargo"]["placementId"], "client_123", + "should use inline bidder params from trustedServer expansion" + ); + } + + #[test] + fn to_openrtb_skips_aps_key_from_slot_bidders_in_pbs_request() { + let slot = make_slot( + "atf_sidebar_ad", + HashMap::from([("aps".to_string(), json!({"slotID": "aps-slot-atf-sidebar"}))]), + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(base_config(), &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("bidder").is_none(), + "should not forward aps key into PBS imp.ext.prebid.bidder" + ); + } } diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 3c9be932e..eca5e70f5 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -162,9 +162,21 @@ pub struct ImpExt { impl ToExt for ImpExt {} -#[derive(Debug, Serialize)] +#[derive(Debug, Default, Serialize)] pub struct PrebidImpExt { + #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] pub bidder: std::collections::HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub storedrequest: Option, +} + +/// PBS imp-level stored request reference. +/// +/// PBS merges the stored imp JSON (keyed by `id`) into the outgoing request, +/// populating bidder params that are not sent inline. +#[derive(Debug, Serialize)] +pub struct ImpStoredRequest { + pub id: String, } #[derive(Debug, Serialize)] diff --git a/creative-opportunities.toml b/creative-opportunities.toml index 0261110a2..3cd27f2b1 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -16,6 +16,10 @@ zone = "atfSidebar" [slot.providers.aps] slot_id = "aps-slot-atf-sidebar" +[slot.providers.pbs.bidders] +mocktioneer = { bid = 2.00 } +criteo = { networkId = 123456, pubid = "123456" } + [[slot]] id = "homepage_header_ad" gam_unit_path = "/88059007/autoblog/homepage" @@ -30,3 +34,7 @@ zone = "header" [slot.providers.aps] slot_id = "aps-slot-homepage-header" + +[slot.providers.pbs.bidders] +mocktioneer = { bid = 2.00 } +criteo = { networkId = 123456, pubid = "123456" } From 5cbf05f1f9f908bbd200a2de52cdec119396a34f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 19:41:03 +0530 Subject: [PATCH 031/395] Fix clippy errors --- crates/trusted-server-core/src/creative_opportunities.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index a7fd99cb6..fa3449fd4 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -342,7 +342,7 @@ mod tests { .get("mocktioneer") .expect("should have mocktioneer bidder"); assert_eq!( - mock_params.get("bid").and_then(|v| v.as_f64()), + mock_params.get("bid").and_then(serde_json::Value::as_f64), Some(2.0), "should wire mocktioneer bid param" ); @@ -351,7 +351,7 @@ mod tests { .get("criteo") .expect("should have criteo bidder"); assert_eq!( - criteo_params.get("networkId").and_then(|v| v.as_i64()), + criteo_params.get("networkId").and_then(serde_json::Value::as_i64), Some(112141), "should wire criteo networkId param" ); From 60011f08b25f8e062366e15c863623767476acd6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 19:46:07 +0530 Subject: [PATCH 032/395] Fix test assertion --- crates/trusted-server-core/src/creative_opportunities.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index fa3449fd4..7a4a10df5 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -351,8 +351,10 @@ mod tests { .get("criteo") .expect("should have criteo bidder"); assert_eq!( - criteo_params.get("networkId").and_then(serde_json::Value::as_i64), - Some(112141), + criteo_params + .get("networkId") + .and_then(serde_json::Value::as_i64), + Some(123456), "should wire criteo networkId param" ); } From cf5091fabfaa76e08aeb34c5905943ae54dd38de Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 20:27:56 +0530 Subject: [PATCH 033/395] Fix double __ts_bids injection --- .../trusted-server-core/src/html_processor.rs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index a3608d9ec..86a8abe79 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -20,6 +20,7 @@ use std::cell::Cell; use std::io; use std::rc::Rc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use lol_html::{ @@ -246,6 +247,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso }); let injected_tsjs = Rc::new(Cell::new(false)); + let injected_bids = Arc::new(AtomicBool::new(false)); let integration_registry = config.integrations.clone(); let script_rewriters = integration_registry.script_rewriters(); let ad_slots_script = config.ad_slots_script.clone(); @@ -291,13 +293,20 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso } }), // Inject __ts_bids before via end_tag_handlers. + // Guard with AtomicBool so the script is only injected once even if + // the origin HTML contains multiple elements (e.g. template fragments). element!("body", { let state = ad_bids_state.clone(); + let injected_bids = injected_bids.clone(); move |el| { let state = state.clone(); + let injected_bids = injected_bids.clone(); if let Some(handlers) = el.end_tag_handlers() { let handler: EndTagHandler<'static> = Box::new( move |end_tag: &mut EndTag<'_>| { + if injected_bids.swap(true, Ordering::SeqCst) { + return Ok(()); + } let script_guard = state.read().expect("should read bid state"); let bids_script = match &*script_guard { Some(s) => s.clone(), @@ -1295,6 +1304,32 @@ mod tests { assert!(bids_pos < body_close_pos, "bids must appear before "); } + #[test] + fn injects_ts_bids_only_once_with_multiple_body_elements() { + let bids_script = + r#""#; + let state = std::sync::Arc::new(std::sync::RwLock::new(Some(bids_script.to_string()))); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + // Malformed HTML with two elements (common in CMS template pages) + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert_eq!( + html.matches("window.__ts_bids").count(), + 1, + "should inject __ts_bids exactly once even with multiple elements" + ); + } + #[test] fn injects_empty_ts_bids_when_state_is_none() { let state = std::sync::Arc::new(std::sync::RwLock::new(None)); From eccfd4538547ddb71b2761669fa7e053d88b4cb0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 21:10:45 +0530 Subject: [PATCH 034/395] Fix max-age cookie issue -> no-store --- crates/trusted-server-core/src/publisher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 193f702c3..c7744ed2e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -668,7 +668,7 @@ pub async fn handle_publisher_request( }; if ad_slots_script.is_some() { - response.set_header(header::CACHE_CONTROL, "private, max-age=0"); + response.set_header(header::CACHE_CONTROL, "private, no-store"); response.remove_header("surrogate-control"); response.remove_header("fastly-surrogate-control"); } From 5bb12d08257da12d3bfa43c85394ee4f4b6198e0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 7 May 2026 13:02:30 +0530 Subject: [PATCH 035/395] Add /__ts/page-bids endpoint for pushState/replaceState --- .../js/lib/src/integrations/gpt/index.test.ts | 16 +- crates/js/lib/src/integrations/gpt/index.ts | 159 ++++++++++++++---- .../trusted-server-adapter-fastly/src/main.rs | 14 +- crates/trusted-server-core/src/publisher.rs | 150 ++++++++++++++++- 4 files changed, 301 insertions(+), 38 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/src/integrations/gpt/index.test.ts index e908a201e..4d501ae34 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/src/integrations/gpt/index.test.ts @@ -13,6 +13,9 @@ type TestWindow = Window & { __ts_ad_slots?: unknown; __ts_bids?: unknown; __tsAdInit?: () => void; + __tsPrevGptSlots?: unknown; + __tsServicesEnabled?: boolean; + __tsSpaHookInstalled?: boolean; }; describe('installTsAdInit', () => { @@ -21,6 +24,9 @@ describe('installTsAdInit', () => { delete (window as TestWindow).__ts_ad_slots; delete (window as TestWindow).__ts_bids; delete (window as TestWindow).__tsAdInit; + delete (window as TestWindow).__tsPrevGptSlots; + delete (window as TestWindow).__tsSpaHookInstalled; + (window as TestWindow).__tsServicesEnabled = false; // jsdom does not implement navigator.sendBeacon; polyfill it for tests if (!('sendBeacon' in navigator)) { Object.defineProperty(navigator, 'sendBeacon', { @@ -203,7 +209,15 @@ describe('installTsAdInit', () => { pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), }; - (window as TestWindow).__ts_ad_slots = []; + (window as TestWindow).__ts_ad_slots = [ + { + id: 'atf', + gam_unit_path: '/123/atf', + div_id: 'atf', + formats: [[300, 250]], + targeting: {}, + }, + ]; (window as TestWindow).__ts_bids = {}; const { installTsAdInit } = await import('./index'); diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index ffb4a687f..06bc7143a 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -45,7 +45,7 @@ interface GoogleTagPubAdsService { getTargeting(key: string): string[]; enableSingleRequest(): void; addEventListener(event: string, fn: (e: SlotRenderEndedEvent) => void): void; - refresh(): void; + refresh(slots?: GoogleTagSlot[]): void; } interface GoogleTag { @@ -56,6 +56,7 @@ interface GoogleTag { size: Array, elementId: string ): GoogleTagSlot | null; + destroySlots(slots?: GoogleTagSlot[]): boolean; enableServices(): void; display(elementId: string): void; _loaded_?: boolean; @@ -202,6 +203,8 @@ type TsWindow = Window & { __ts_ad_slots?: TsAdSlot[]; __ts_bids?: Record; __tsAdInit?: () => void; + __tsPrevGptSlots?: GoogleTagSlot[]; + __tsServicesEnabled?: boolean; }; /** @@ -212,6 +215,9 @@ type TsWindow = Window & { * targeting to GPT slots, sets the `ts_initial` sentinel, registers * `slotRenderEnded` to fire both nurl and burl via sendBeacon when our * specific Prebid bid wins the GAM line item match, then calls refresh(). + * + * Idempotent: destroys previously created TS-managed slots before redefining them, + * so it is safe to call again after SPA navigation updates `__ts_ad_slots`/`__ts_bids`. */ export function installTsAdInit(): void { const w = window as TsWindow; @@ -222,46 +228,128 @@ export function installTsAdInit(): void { if (!g) return; g.cmd?.push(() => { - slots - .map((slot) => { - const gptSlot = g.defineSlot?.( - slot.gam_unit_path, - slot.formats as Array, - slot.div_id - ); - if (!gptSlot) return null; - gptSlot.addService(g.pubads!()); - Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v)); - const bid = bids[slot.id] ?? {}; - (['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { - if (bid[key]) gptSlot.setTargeting(key, bid[key]!); - }); - gptSlot.setTargeting('ts_initial', '1'); - return { id: slot.id, gptSlot }; - }) - .filter(Boolean); - - g.pubads!().enableSingleRequest(); - g.enableServices?.(); - - g.pubads!().addEventListener?.('slotRenderEnded', (event: SlotRenderEndedEvent) => { - const slotId: string = event.slot?.getSlotElementId?.() ?? ''; - const bid = bids[slotId] ?? {}; - const ourBidWon = - !event.isEmpty && - bid.hb_adid && - event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid; - if (ourBidWon) { - if (bid.nurl) navigator.sendBeacon(bid.nurl); - if (bid.burl) navigator.sendBeacon(bid.burl); - } + // Destroy previously defined TS slots before redefining for the new page. + if (w.__tsPrevGptSlots && w.__tsPrevGptSlots.length > 0) { + g.destroySlots?.(w.__tsPrevGptSlots); + w.__tsPrevGptSlots = []; + } + + const newSlots: GoogleTagSlot[] = []; + + slots.forEach((slot) => { + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats as Array, + slot.div_id + ); + if (!gptSlot) return; + gptSlot.addService(g.pubads!()); + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v)); + const bid = bids[slot.id] ?? {}; + (['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!); + }); + gptSlot.setTargeting('ts_initial', '1'); + newSlots.push(gptSlot); }); - g.pubads!().refresh(); + w.__tsPrevGptSlots = newSlots; + + // enableSingleRequest and enableServices must only be called once per page load. + if (!w.__tsServicesEnabled) { + g.pubads!().enableSingleRequest(); + g.enableServices?.(); + w.__tsServicesEnabled = true; + + g.pubads!().addEventListener?.('slotRenderEnded', (event: SlotRenderEndedEvent) => { + const slotId: string = event.slot?.getSlotElementId?.() ?? ''; + const bid = (w.__ts_bids ?? {})[slotId] ?? {}; + const ourBidWon = + !event.isEmpty && + bid.hb_adid && + event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid; + if (ourBidWon) { + if (bid.nurl) navigator.sendBeacon(bid.nurl); + if (bid.burl) navigator.sendBeacon(bid.burl); + } + }); + } + + if (newSlots.length > 0) { + g.pubads!().refresh(newSlots); + } }); }; } +interface PageBidsResponse { + slots: TsAdSlot[]; + bids: Record; +} + +/** + * Install SPA navigation hook. + * + * Patches `history.pushState` and `history.replaceState`, and listens to + * `popstate`, so that after each client-side route change the trusted server + * fetches fresh slots + bids from `/__ts/page-bids?path=`, updates + * `window.__ts_ad_slots` / `window.__ts_bids`, and calls `window.__tsAdInit()`. + * + * Idempotent: guarded by `window.__tsSpaHookInstalled` so multiple calls are safe. + */ +export function installSpaAuctionHook(): void { + if (typeof window === 'undefined') return; + const win = window as TsWindow & { __tsSpaHookInstalled?: boolean }; + if (win.__tsSpaHookInstalled) return; + win.__tsSpaHookInstalled = true; + + let inflight: AbortController | null = null; + + async function onNavigate(path: string): Promise { + inflight?.abort(); + const controller = new AbortController(); + inflight = controller; + + try { + const res = await fetch(`/__ts/page-bids?path=${encodeURIComponent(path)}`, { + credentials: 'include', + signal: controller.signal, + }); + if (!res.ok) return; + const data = (await res.json()) as PageBidsResponse; + win.__ts_ad_slots = data.slots; + win.__ts_bids = data.bids; + win.__tsAdInit?.(); + } catch (err) { + if (err instanceof DOMException && err.name === 'AbortError') return; + log.warn('SPA auction hook: fetch failed', err); + } + } + + function patchHistoryMethod(method: 'pushState' | 'replaceState'): void { + const original = history[method].bind(history); + history[method] = function ( + state: unknown, + unused: string, + url?: string | URL | null + ): void { + const prevPath = location.pathname; + original(state, unused, url); + const newPath = url ? new URL(String(url), location.href).pathname : location.pathname; + if (newPath !== prevPath) { + void onNavigate(newPath); + } + }; + } + + patchHistoryMethod('pushState'); + patchHistoryMethod('replaceState'); + + window.addEventListener('popstate', () => { + void onNavigate(location.pathname); + }); +} + /** * Register the slim-Prebid lazy loader. Fires after window.load — off the * critical path. slim-Prebid handles refresh auctions and userID module @@ -300,4 +388,5 @@ if (typeof window !== 'undefined') { } installTsAdInit(); + installSpaAuctionHook(); } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 74414220b..55af1468e 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -19,7 +19,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - handle_publisher_request, handle_tsjs_dynamic, stream_publisher_body, PublisherResponse, + handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, stream_publisher_body, + PublisherResponse, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -194,6 +195,17 @@ async fn route_request( } } + // SPA/CSR navigation endpoint — returns slots + bids JSON for the given path + (Method::GET, "/__ts/page-bids") => { + match runtime_services_for_consent_route(settings, runtime_services) { + Ok(publisher_services) => { + handle_page_bids(settings, orchestrator, &publisher_services, slots_file, req) + .await + } + Err(e) => Err(e), + } + } + // tsjs endpoints (Method::GET, "/first-party/proxy") => { handle_first_party_proxy(settings, runtime_services, req).await diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index c7744ed2e..ec4f4a227 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -668,7 +668,7 @@ pub async fn handle_publisher_request( }; if ad_slots_script.is_some() { - response.set_header(header::CACHE_CONTROL, "private, no-store"); + response.set_header(header::CACHE_CONTROL, "private, max-age=0"); response.remove_header("surrogate-control"); response.remove_header("fastly-surrogate-control"); } @@ -990,6 +990,154 @@ fn apply_ec_headers( } } +/// Handle `GET /__ts/page-bids?path=` — server-side auction for SPA navigation. +/// +/// Matches creative opportunity slots for the given path, runs a server-side +/// auction (APS + PBS), and returns the slot definitions and winning bids as JSON. +/// Called by the client-side SPA navigation hook after `pushState` / `popstate`. +/// +/// # Errors +/// +/// Returns [`TrustedServerError`] if cookie parsing or EC ID generation fails. +pub async fn handle_page_bids( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, + slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, + req: Request, +) -> Result> { + let Some(co_config) = &settings.creative_opportunities else { + return Ok(Response::from_status(StatusCode::NOT_FOUND) + .with_body_text_plain("Creative opportunities not configured")); + }; + + let path_param = req + .get_url() + .query_pairs() + .find(|(k, _)| k == "path") + .map(|(_, v)| v.into_owned()) + .unwrap_or_else(|| "/".to_string()); + + let matched_slots: Vec<_> = + crate::creative_opportunities::match_slots(&slots_file.slots, &path_param) + .into_iter() + .cloned() + .collect(); + + let request_info = crate::http_util::RequestInfo::from_request(&req, &services.client_info); + let cookie_jar = handle_request_cookies(&req)?; + let ec_id = get_or_generate_ec_id(settings, services, &req)?; + let geo = services + .geo() + .lookup(services.client_info.client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + let consent_context = build_consent_context(&ConsentPipelineInput { + jar: cookie_jar.as_ref(), + req: &req, + config: &settings.consent, + geo: geo.as_ref(), + ec_id: Some(ec_id.as_str()), + kv_store: settings + .consent + .consent_store + .as_deref() + .map(|_| services.kv_store()), + }); + + let consent_allows_auction = consent_context + .tcf + .as_ref() + .is_some_and(|tcf| tcf.has_purpose_consent(1)); + + let winning_bids = if !matched_slots.is_empty() && consent_allows_auction { + let mut auction_request = build_auction_request( + &matched_slots, + &ec_id, + &consent_context, + &request_info, + co_config, + ); + let page_url = format!( + "{}://{}{}", + request_info.scheme, request_info.host, path_param + ); + auction_request.publisher.page_url = Some(page_url.clone()); + if let Some(ref mut site) = auction_request.site { + site.page = page_url; + } + let timeout_ms = co_config + .auction_timeout_ms + .unwrap_or(settings.auction.timeout_ms); + let placeholder_req = fastly::Request::get("https://placeholder.invalid/"); + let auction_context = AuctionContext { + settings, + request: &placeholder_req, + client_info: services.client_info(), + timeout_ms, + provider_responses: None, + services, + }; + match orchestrator + .run_auction(&auction_request, &auction_context, services) + .await + { + Ok(result) => result.winning_bids, + Err(e) => { + log::warn!("page-bids auction failed: {e:?}"); + std::collections::HashMap::new() + } + } + } else { + std::collections::HashMap::new() + }; + + let bid_map = build_bid_map(&winning_bids, co_config.price_granularity); + + let slots_json: Vec = matched_slots + .iter() + .map(|slot| { + let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); + let div_id = slot.resolved_div_id(); + let formats: Vec = slot + .formats + .iter() + .map(|f| serde_json::json!([f.width, f.height])) + .collect(); + let targeting: serde_json::Map = slot + .targeting + .iter() + .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) + .collect(); + serde_json::json!({ + "id": slot.id, + "gam_unit_path": gam_path, + "div_id": div_id, + "formats": formats, + "targeting": targeting, + }) + }) + .collect(); + + let body = serde_json::json!({ + "slots": slots_json, + "bids": bid_map, + }); + + let json_str = serde_json::to_string(&body).change_context(TrustedServerError::Proxy { + message: "Failed to serialize page-bids response".to_string(), + })?; + + let mut response = Response::from_status(StatusCode::OK); + response.set_header(header::CONTENT_TYPE, "application/json"); + response.set_header(header::CACHE_CONTROL, "private, no-store"); + response.set_body(json_str); + + Ok(response) +} + #[cfg(test)] mod tests { use super::*; From 982fa3edbf8ed881797c6dff4aacd51d2878d68b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 7 May 2026 13:04:19 +0530 Subject: [PATCH 036/395] Fix format ts --- crates/js/lib/src/integrations/gpt/index.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 06bc7143a..bf9fc99de 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -328,11 +328,7 @@ export function installSpaAuctionHook(): void { function patchHistoryMethod(method: 'pushState' | 'replaceState'): void { const original = history[method].bind(history); - history[method] = function ( - state: unknown, - unused: string, - url?: string | URL | null - ): void { + history[method] = function (state: unknown, unused: string, url?: string | URL | null): void { const prevPath = location.pathname; original(state, unused, url); const newPath = url ? new URL(String(url), location.href).pathname : location.pathname; From 77d3c4a2e92f7d098c901322637463657bdb01ee Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 7 May 2026 15:46:49 +0530 Subject: [PATCH 037/395] =?UTF-8?q?=5F=5FtsDivToSlotId=20now=20replaced=20?= =?UTF-8?q?per=20navigation=20(not=20merged)=20=E2=80=94=20stale=20div=5Fi?= =?UTF-8?q?d=20entries=20from=20destroyed=20slots=20no=20longer=20persist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../js/lib/src/integrations/gpt/index.test.ts | 138 ++++++++++++++++-- crates/js/lib/src/integrations/gpt/index.ts | 17 ++- 2 files changed, 138 insertions(+), 17 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/src/integrations/gpt/index.test.ts index 4d501ae34..87455591e 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/src/integrations/gpt/index.test.ts @@ -16,6 +16,7 @@ type TestWindow = Window & { __tsPrevGptSlots?: unknown; __tsServicesEnabled?: boolean; __tsSpaHookInstalled?: boolean; + __tsDivToSlotId?: Record; }; describe('installTsAdInit', () => { @@ -26,6 +27,7 @@ describe('installTsAdInit', () => { delete (window as TestWindow).__tsAdInit; delete (window as TestWindow).__tsPrevGptSlots; delete (window as TestWindow).__tsSpaHookInstalled; + delete (window as TestWindow).__tsDivToSlotId; (window as TestWindow).__tsServicesEnabled = false; // jsdom does not implement navigator.sendBeacon; polyfill it for tests if (!('sendBeacon' in navigator)) { @@ -41,7 +43,7 @@ describe('installTsAdInit', () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), - getSlotElementId: vi.fn().mockReturnValue('atf'), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), getTargeting: vi.fn().mockReturnValue(['abc']), }; const mockPubads = { @@ -57,15 +59,15 @@ describe('installTsAdInit', () => { }; (window as TestWindow).__ts_ad_slots = [ { - id: 'atf', + id: 'atf_sidebar_ad', gam_unit_path: '/123/atf', - div_id: 'atf', + div_id: 'div-atf-sidebar', formats: [[300, 250]], targeting: { pos: 'atf' }, }, ]; (window as TestWindow).__ts_bids = { - atf: { + atf_sidebar_ad: { hb_pb: '1.00', hb_bidder: 'kargo', hb_adid: 'abc', @@ -96,7 +98,7 @@ describe('installTsAdInit', () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), - getSlotElementId: vi.fn().mockReturnValue('atf'), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), getTargeting: vi.fn().mockReturnValue(['abc']), }; const mockPubads = { @@ -114,15 +116,15 @@ describe('installTsAdInit', () => { }; (window as TestWindow).__ts_ad_slots = [ { - id: 'atf', + id: 'atf_sidebar_ad', gam_unit_path: '/123/atf', - div_id: 'atf', + div_id: 'div-atf-sidebar', formats: [[300, 250]], targeting: {}, }, ]; (window as TestWindow).__ts_bids = { - atf: { + atf_sidebar_ad: { hb_pb: '1.00', hb_bidder: 'kargo', hb_adid: 'abc', @@ -143,6 +145,64 @@ describe('installTsAdInit', () => { beaconSpy.mockRestore(); }); + it('fires beacons for APS bid (no hb_adid) when ad renders in our slot', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; + + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).__ts_ad_slots = [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ]; + (window as TestWindow).__ts_bids = { + atf_sidebar_ad: { + hb_pb: '1.50', + hb_bidder: 'aps', + nurl: 'https://aps/win', + burl: 'https://aps/bill', + }, + }; + + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as TestWindow).__tsAdInit!(); + + expect(capturedListener).toBeDefined(); + capturedListener!({ isEmpty: false, slot: mockSlot }); + + expect(beaconSpy).toHaveBeenCalledWith('https://aps/win'); + expect(beaconSpy).toHaveBeenCalledWith('https://aps/bill'); + + beaconSpy.mockClear(); + capturedListener!({ isEmpty: true, slot: mockSlot }); + expect(beaconSpy).not.toHaveBeenCalled(); + + beaconSpy.mockRestore(); + }); + it('does not fire nurl/burl when bid did not win GAM line item', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); let capturedListener: ((e: SlotRenderEvent) => void) | undefined; @@ -150,7 +210,7 @@ describe('installTsAdInit', () => { const mockSlotNoMatch = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), - getSlotElementId: vi.fn().mockReturnValue('atf'), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), getTargeting: vi.fn().mockReturnValue(['OTHER_BID_ID']), }; const mockPubads = { @@ -168,15 +228,15 @@ describe('installTsAdInit', () => { }; (window as TestWindow).__ts_ad_slots = [ { - id: 'atf', + id: 'atf_sidebar_ad', gam_unit_path: '/123/atf', - div_id: 'atf', + div_id: 'div-atf-sidebar', formats: [[300, 250]], targeting: {}, }, ]; (window as TestWindow).__ts_bids = { - atf: { + atf_sidebar_ad: { hb_pb: '1.00', hb_bidder: 'kargo', hb_adid: 'abc', @@ -194,6 +254,56 @@ describe('installTsAdInit', () => { beaconSpy.mockRestore(); }); + it('does not fire beacons for slotRenderEnded on slots not owned by TS', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; + + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue(['abc']), + }; + const arenaSlot = { + getSlotElementId: () => 'arena-owned-div', + getTargeting: () => [], + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).__ts_ad_slots = [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ]; + (window as TestWindow).__ts_bids = { + atf_sidebar_ad: { hb_pb: '1.00', hb_bidder: 'kargo', hb_adid: 'abc' }, + }; + + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as TestWindow).__tsAdInit!(); + + capturedListener!({ isEmpty: false, slot: arenaSlot }); + + expect(beaconSpy).not.toHaveBeenCalled(); + beaconSpy.mockRestore(); + }); + it('calls refresh even when __ts_bids is empty (graceful fallback)', async () => { const mockPubads = { enableSingleRequest: vi.fn(), @@ -211,9 +321,9 @@ describe('installTsAdInit', () => { }; (window as TestWindow).__ts_ad_slots = [ { - id: 'atf', + id: 'atf_sidebar_ad', gam_unit_path: '/123/atf', - div_id: 'atf', + div_id: 'div-atf-sidebar', formats: [[300, 250]], targeting: {}, }, diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index bf9fc99de..fee79c1b6 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -205,6 +205,7 @@ type TsWindow = Window & { __tsAdInit?: () => void; __tsPrevGptSlots?: GoogleTagSlot[]; __tsServicesEnabled?: boolean; + __tsDivToSlotId?: Record; }; /** @@ -235,6 +236,7 @@ export function installTsAdInit(): void { } const newSlots: GoogleTagSlot[] = []; + const divToSlotId: Record = {}; slots.forEach((slot) => { const gptSlot = g.defineSlot?.( @@ -250,10 +252,13 @@ export function installTsAdInit(): void { if (bid[key]) gptSlot.setTargeting(key, bid[key]!); }); gptSlot.setTargeting('ts_initial', '1'); + divToSlotId[slot.div_id] = slot.id; newSlots.push(gptSlot); }); w.__tsPrevGptSlots = newSlots; + // Replace (not merge) so destroyed slots from previous navigation don't linger. + w.__tsDivToSlotId = divToSlotId; // enableSingleRequest and enableServices must only be called once per page load. if (!w.__tsServicesEnabled) { @@ -262,12 +267,18 @@ export function installTsAdInit(): void { w.__tsServicesEnabled = true; g.pubads!().addEventListener?.('slotRenderEnded', (event: SlotRenderEndedEvent) => { - const slotId: string = event.slot?.getSlotElementId?.() ?? ''; + const divId: string = event.slot?.getSlotElementId?.() ?? ''; + const slotId = (w.__tsDivToSlotId ?? {})[divId]; + if (!slotId) return; const bid = (w.__ts_bids ?? {})[slotId] ?? {}; + // Prebid: compare hb_adid targeting to verify the specific creative won. + // APS: no hb_adid equivalent — fires if bidder exists and slot is non-empty. + // Known limitation: APS path may over-fire if a non-APS line item wins. const ourBidWon = !event.isEmpty && - bid.hb_adid && - event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid; + (bid.hb_adid + ? event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + : !!bid.hb_bidder); if (ourBidWon) { if (bid.nurl) navigator.sendBeacon(bid.nurl); if (bid.burl) navigator.sendBeacon(bid.burl); From 38c8bf17701dea1a76ba1344620f726a0a18711b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 7 May 2026 17:20:49 +0530 Subject: [PATCH 038/395] Update timeout for mocktioneer --- trusted-server.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trusted-server.toml b/trusted-server.toml index 43e090fea..d17e86479 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -172,7 +172,7 @@ allowed_context_keys = ["permutive_segments"] enabled = true pub_id = "test-pub" endpoint = "https://origin-mocktioneer.cdintel.com/e/dtb/bid" -timeout_ms = 1000 +timeout_ms = 400 [integrations.google_tag_manager] enabled = false @@ -182,7 +182,7 @@ container_id = "GTM-XXXXXX" [integrations.adserver_mock] enabled = true endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" -timeout_ms = 1000 +timeout_ms = 400 # Map auction-request context keys to mediation URL query parameters. # Each key is a context key from the JS client; the value becomes the @@ -192,6 +192,6 @@ permutive_segments = "permutive" [creative_opportunities] gam_network_id = "88059007" -auction_timeout_ms = 1500 +auction_timeout_ms = 500 price_granularity = "dense" From e32bfa556e99d1fb225876c286e2fb7164317c9f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 7 May 2026 17:28:59 +0530 Subject: [PATCH 039/395] Revert with updated tiomeout --- trusted-server.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trusted-server.toml b/trusted-server.toml index d17e86479..43e090fea 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -172,7 +172,7 @@ allowed_context_keys = ["permutive_segments"] enabled = true pub_id = "test-pub" endpoint = "https://origin-mocktioneer.cdintel.com/e/dtb/bid" -timeout_ms = 400 +timeout_ms = 1000 [integrations.google_tag_manager] enabled = false @@ -182,7 +182,7 @@ container_id = "GTM-XXXXXX" [integrations.adserver_mock] enabled = true endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" -timeout_ms = 400 +timeout_ms = 1000 # Map auction-request context keys to mediation URL query parameters. # Each key is a context key from the JS client; the value becomes the @@ -192,6 +192,6 @@ permutive_segments = "permutive" [creative_opportunities] gam_network_id = "88059007" -auction_timeout_ms = 500 +auction_timeout_ms = 1500 price_granularity = "dense" From b1e74c986ec44f78053d4ef2dbadfc34697bb824 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 9 May 2026 14:45:34 +0530 Subject: [PATCH 040/395] Wip: Align with the spec --- .../trusted-server-adapter-fastly/src/main.rs | 18 +- .../src/auction/orchestrator.rs | 338 ++++++++++++++++++ .../src/creative_opportunities.rs | 16 +- .../trusted-server-core/src/html_processor.rs | 47 ++- crates/trusted-server-core/src/publisher.rs | 329 +++++++++++++++-- trusted-server.toml | 6 + 6 files changed, 710 insertions(+), 44 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 55af1468e..895299f54 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -19,7 +19,7 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, stream_publisher_body, + handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, stream_publisher_body_async, PublisherResponse, }; use trusted_server_core::request_signing::{ @@ -250,18 +250,26 @@ async fn route_request( Ok(PublisherResponse::Stream { mut response, body, - params, + mut params, }) => { // Streaming path: finalize headers, then stream body to client. + // TTFB happens at stream_to_client() — SSP bids are already + // in-flight in Fastly's native layer (dispatched before origin wait). finalize_response(settings, geo_info.as_ref(), &mut response); let mut streaming_body = response.stream_to_client(); - if let Err(e) = stream_publisher_body( + // stream_publisher_body_async falls back to the sync path + // when no auction was dispatched (dispatched_auction is None). + let stream_result = stream_publisher_body_async( body, &mut streaming_body, - ¶ms, + &mut *params, settings, integration_registry, - ) { + orchestrator, + &publisher_services, + ) + .await; + if let Err(e) = stream_result { // Headers already committed. Log and abort — client // sees a truncated response. Standard proxy behavior. log::error!("Streaming processing failed: {e:?}"); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 0a52b07c8..953ed6a04 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -13,6 +13,23 @@ use super::config::AuctionConfig; use super::provider::AuctionProvider; use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus}; +/// In-flight auction requests dispatched to SSP backends. +/// +/// Created by [`AuctionOrchestrator::dispatch_auction`] and consumed by +/// [`AuctionOrchestrator::collect_dispatched_auction`]. Carrying this handle +/// across `pending_origin.wait()` lets origin response and SSP HTTP requests +/// race in Fastly's native layer, enabling TTFB ≈ origin latency rather than +/// TTFB ≈ auction timeout. +pub struct DispatchedAuction { + pending_requests: Vec, + backend_to_provider: HashMap)>, + auction_start: Instant, + timeout_ms: u32, + floor_prices: HashMap, + /// Carried so the mediator call in collect can pass it as the auction request. + request: AuctionRequest, +} + /// Compute the remaining time budget from a deadline. /// /// Returns the number of milliseconds left before `timeout_ms` is exceeded, @@ -584,6 +601,327 @@ impl AuctionOrchestrator { }) } + /// Dispatch SSP bid requests without blocking WASM. + /// + /// Calls each enabled provider's [`AuctionProvider::request_bids`] (which + /// internally calls Fastly's `send_async`), then returns immediately with a + /// [`DispatchedAuction`] token. The Fastly host begins the SSP round-trips + /// while WASM continues to `pending_origin.wait()`. + /// + /// Returns `None` when no providers are configured or all providers are + /// disabled / over budget. The caller should fall back to the synchronous + /// `run_auction` path. + #[must_use] + pub fn dispatch_auction( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Option { + let provider_names = self.config.provider_names(); + if provider_names.is_empty() { + return None; + } + + let auction_start = Instant::now(); + let mut backend_to_provider: HashMap)> = + HashMap::new(); + let mut pending_requests: Vec = Vec::new(); + + for provider_name in provider_names { + let provider = match self.providers.get(provider_name) { + Some(p) => p, + None => { + log::warn!("Provider '{}' not registered, skipping", provider_name); + continue; + } + }; + + if !provider.is_enabled() { + log::debug!("Provider '{}' is disabled, skipping", provider.provider_name()); + continue; + } + + let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); + let effective_timeout = remaining_ms.min(provider.timeout_ms()); + + if effective_timeout == 0 { + log::warn!( + "Auction timeout ({}ms) exhausted before launching '{}' — skipping", + context.timeout_ms, + provider.provider_name() + ); + continue; + } + + let backend_name = match provider.backend_name(effective_timeout) { + Some(name) => name, + None => { + log::warn!("Provider '{}' has no backend_name, skipping", provider.provider_name()); + continue; + } + }; + + let provider_context = AuctionContext { + settings: context.settings, + request: context.request, + client_info: context.client_info, + timeout_ms: effective_timeout, + provider_responses: context.provider_responses, + services: context.services, + }; + + let start_time = Instant::now(); + match provider.request_bids(request, &provider_context) { + Ok(pending) => { + log::info!( + "Dispatching bid request to '{}' (backend: {}, budget: {}ms)", + provider.provider_name(), + backend_name, + effective_timeout + ); + backend_to_provider.insert( + backend_name.clone(), + (provider.provider_name().to_string(), start_time, Arc::clone(provider)), + ); + pending_requests + .push(PlatformPendingRequest::new(pending).with_backend_name(backend_name)); + } + Err(e) => { + log::warn!( + "Provider '{}' failed to dispatch request: {:?}", + provider.provider_name(), + e + ); + } + } + } + + if pending_requests.is_empty() { + return None; + } + + log::info!( + "Dispatched {} SSP requests (timeout: {}ms); Fastly host will race them against origin", + pending_requests.len(), + context.timeout_ms + ); + + Some(DispatchedAuction { + pending_requests, + backend_to_provider, + auction_start, + timeout_ms: context.timeout_ms, + floor_prices: self.floor_prices_by_slot(request), + request: request.clone(), + }) + } + + /// Collect bid responses from a previously-dispatched auction. + /// + /// Runs the select-loop phase (equivalent to Phase 2 of + /// `run_providers_parallel`) and, if the orchestrator has a mediator + /// configured, forwards collected bids to it. The overall auction deadline + /// is enforced from `dispatched.auction_start`. + /// + /// On any error or partial failure the method returns the best available + /// result rather than propagating — the caller should still inject the + /// winning bids even if some providers timed out. + pub async fn collect_dispatched_auction( + &self, + dispatched: DispatchedAuction, + services: &RuntimeServices, + context: &AuctionContext<'_>, + ) -> OrchestrationResult { + let DispatchedAuction { + pending_requests, + mut backend_to_provider, + auction_start, + timeout_ms, + floor_prices, + request, + } = dispatched; + + let deadline = Duration::from_millis(u64::from(timeout_ms)); + + log::info!( + "Collecting {} in-flight SSP responses (timeout: {}ms remaining: {}ms)", + pending_requests.len(), + timeout_ms, + remaining_budget_ms(auction_start, timeout_ms), + ); + + let mut responses: Vec = Vec::new(); + let mut remaining = pending_requests; + + while !remaining.is_empty() { + let select_result = match services + .http_client() + .select(remaining) + .await + .change_context(TrustedServerError::Auction { + message: "HTTP select failed".to_string(), + }) { + Ok(r) => r, + Err(e) => { + log::warn!("select() failed during auction collection: {:?}", e); + break; + } + }; + remaining = select_result.remaining; + + match select_result.ready { + Ok(platform_response) => { + let backend_name = platform_response.backend_name.clone().unwrap_or_default(); + if let Some((provider_name, start_time, provider)) = + backend_to_provider.remove(&backend_name) + { + let response_time_ms = start_time.elapsed().as_millis() as u64; + match platform_response_to_fastly(platform_response) { + Ok(response) => match provider.parse_response(response, response_time_ms) { + Ok(auction_response) => { + log::info!( + "Provider '{}' returned {} bids ({}ms)", + auction_response.provider, + auction_response.bids.len(), + auction_response.response_time_ms + ); + responses.push(auction_response); + } + Err(e) => { + log::warn!("Provider '{}' parse failed: {:?}", provider_name, e); + responses.push(AuctionResponse::error(&provider_name, response_time_ms)); + } + }, + Err(e) => { + log::warn!("Provider '{}' unsupported body: {:?}", provider_name, e); + responses.push(AuctionResponse::error(&provider_name, response_time_ms)); + } + } + } else { + log::warn!("Received response from unknown backend '{}', ignoring", backend_name); + } + } + Err(e) => { + log::warn!("A provider request failed during collection: {:?}", e); + } + } + + if auction_start.elapsed() >= deadline && !remaining.is_empty() { + log::warn!( + "Auction timeout ({}ms) reached, dropping {} remaining request(s)", + timeout_ms, + remaining.len() + ); + break; + } + } + + let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { + match self.providers.get(mediator_name.as_str()) { + Some(mediator) => { + let remaining_ms = remaining_budget_ms(auction_start, timeout_ms); + if remaining_ms == 0 { + log::warn!("Auction timeout exhausted during bidding — skipping mediator"); + let winning = self.select_winning_bids(&responses, &floor_prices); + return OrchestrationResult { + provider_responses: responses, + mediator_response: None, + winning_bids: winning, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: HashMap::new(), + }; + } + let placeholder = fastly::Request::get("https://placeholder.invalid/"); + let mediator_context = AuctionContext { + settings: context.settings, + request: &placeholder, + client_info: context.client_info, + timeout_ms: remaining_ms, + provider_responses: Some(&responses), + services: context.services, + }; + match mediator.request_bids(&request, &mediator_context) { + Ok(pending) => { + let platform_resp = services + .http_client() + .wait(PlatformPendingRequest::new(pending)) + .await; + match platform_resp.change_context(TrustedServerError::Auction { + message: format!("Mediator {} request failed", mediator.provider_name()), + }) { + Ok(platform_resp) => { + match platform_response_to_fastly(platform_resp).change_context( + TrustedServerError::Auction { + message: format!("Mediator {} unsupported body", mediator.provider_name()), + }, + ) { + Ok(response) => { + let response_time_ms = + remaining_ms as u64 - remaining_budget_ms(auction_start, timeout_ms) as u64; + match mediator.parse_response(response, response_time_ms) { + Ok(mediator_resp) => { + let winning = mediator_resp + .bids + .iter() + .filter_map(|bid| { + if bid.price.is_none() { + log::warn!( + "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", + mediator.provider_name(), + bid.slot_id + ); + None + } else { + Some((bid.slot_id.clone(), bid.clone())) + } + }) + .collect(); + let winning = self.apply_floor_prices(winning, &floor_prices); + (Some(mediator_resp), winning) + } + Err(e) => { + log::warn!("Mediator '{}' parse failed: {:?}", mediator.provider_name(), e); + let winning = self.select_winning_bids(&responses, &floor_prices); + (None, winning) + } + } + } + Err(e) => { + log::warn!("Mediator body error: {:?}", e); + (None, self.select_winning_bids(&responses, &floor_prices)) + } + } + } + Err(e) => { + log::warn!("Mediator request failed: {:?}", e); + (None, self.select_winning_bids(&responses, &floor_prices)) + } + } + } + Err(e) => { + log::warn!("Mediator '{}' failed to dispatch: {:?}", mediator.provider_name(), e); + (None, self.select_winning_bids(&responses, &floor_prices)) + } + } + } + None => { + log::warn!("Mediator '{}' not registered", mediator_name); + (None, self.select_winning_bids(&responses, &floor_prices)) + } + } + } else { + (None, self.select_winning_bids(&responses, &floor_prices)) + }; + + OrchestrationResult { + provider_responses: responses, + mediator_response, + winning_bids, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: HashMap::new(), + } + } + /// Check if orchestrator is enabled. #[must_use] pub fn is_enabled(&self) -> bool { diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 7a4a10df5..12957d4b8 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -18,7 +18,21 @@ use crate::price_bucket::PriceGranularity; pub struct CreativeOpportunitiesConfig { /// GAM network ID used to build default unit paths. pub gam_network_id: String, - /// Auction timeout in milliseconds. + /// Maximum time in milliseconds to wait for the server-side auction before + /// closing the response body. + /// + /// The auction runs concurrently with HTML body streaming. Body content + /// above `` has already been delivered and painted before the hold + /// begins, so **FCP is not affected**. What this timeout bounds is the slip + /// on `DOMContentLoaded` and `window.load`: third-party scripts that hook + /// those events fire later by at most this duration. + /// + /// The worst case is a cache-hit page where the origin drains in <50 ms + /// but the auction takes the full timeout — the browser sits idle waiting + /// for ``. 500 ms is the recommended default and the hard upper + /// bound on DCL slip the publisher is willing to accept. + /// + /// When absent, falls back to `[auction].timeout_ms` from global config. #[serde(default)] pub auction_timeout_ms: Option, /// Price granularity for header-bidding price bucketing. diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 86a8abe79..26978cef5 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -292,13 +292,21 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso Ok(()) } }), - // Inject __ts_bids before via end_tag_handlers. + // Inject __ts_bids before via end_tag_handlers — only when + // slots matched this URL. When no slots matched, skip injection entirely + // so the publisher's existing client-side Prebid/GPT flow is unmodified + // (dual-mode rollout: calling __tsAdInit with empty slots would invoke + // enableSingleRequest/enableServices and conflict with the publisher's GPT init). // Guard with AtomicBool so the script is only injected once even if // the origin HTML contains multiple elements (e.g. template fragments). element!("body", { let state = ad_bids_state.clone(); let injected_bids = injected_bids.clone(); + let has_slots = ad_slots_script.is_some(); move |el| { + if !has_slots { + return Ok(()); + } let state = state.clone(); let injected_bids = injected_bids.clone(); if let Some(handlers) = el.end_tag_handlers() { @@ -1285,7 +1293,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: None, + ad_slots_script: Some("".to_string()), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1314,7 +1322,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: None, + ad_slots_script: Some("".to_string()), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1331,14 +1339,16 @@ mod tests { } #[test] - fn injects_empty_ts_bids_when_state_is_none() { + fn injects_empty_ts_bids_when_slots_matched_but_auction_returned_nothing() { + // Slots matched (ad_slots_script is Some) but auction task never wrote a result + // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. let state = std::sync::Arc::new(std::sync::RwLock::new(None)); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: None, + ad_slots_script: Some("".to_string()), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1348,7 +1358,32 @@ mod tests { let html = std::str::from_utf8(&output).expect("should be utf8"); assert!( html.contains("__ts_bids=JSON.parse(\"{}\")"), - "should inject empty bids on None state" + "should inject empty bids fallback when auction produced nothing" + ); + } + + #[test] + fn does_not_inject_ts_bids_when_no_slots_matched() { + // No slots matched this URL — ad_slots_script is None. __ts_bids must be + // omitted entirely so the publisher's existing client-side GPT flow is + // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). + let state = std::sync::Arc::new(std::sync::RwLock::new(None)); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + !html.contains("__ts_bids"), + "should NOT inject __ts_bids when no slots matched" ); } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index ec4f4a227..4a39c9623 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -18,7 +18,7 @@ use error_stack::{Report, ResultExt}; use fastly::http::{header, StatusCode}; use fastly::{Body, Request, Response}; -use crate::auction::orchestrator::AuctionOrchestrator; +use crate::auction::orchestrator::{AuctionOrchestrator, DispatchedAuction}; use crate::auction::types::{ AuctionContext, AuctionRequest, Bid, PublisherInfo, SiteInfo, UserInfo, }; @@ -31,7 +31,7 @@ use crate::error::TrustedServerError; use crate::http_util::{serve_static_with_etag, RequestInfo}; use crate::integrations::IntegrationRegistry; use crate::platform::RuntimeServices; -use crate::price_bucket::price_bucket; +use crate::price_bucket::{price_bucket, PriceGranularity}; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, StreamingPipeline}; @@ -301,8 +301,9 @@ pub enum PublisherResponse { response: Response, /// Origin body to be piped through the streaming pipeline. body: Body, - /// Parameters for `process_response_streaming`. - params: OwnedProcessResponseParams, + /// Parameters for `process_response_streaming`. Boxed to keep this + /// variant's on-stack size comparable to the other variants. + params: Box, }, /// Non-processable 2xx response (images, fonts, video). The adapter must /// reattach the body via `response.set_body(body)` before returning. @@ -407,6 +408,12 @@ pub struct OwnedProcessResponseParams { pub(crate) content_type: String, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: Arc>>, + /// In-flight SSP bids dispatched before `pending_origin.wait()`. + /// The streaming phase collects these and writes bids to `ad_bids_state` + /// before processing the last body chunk, so `` injection sees live bids. + pub(crate) dispatched_auction: Option, + /// Price granularity used to bucket bids when building `__ts_bids`. + pub(crate) price_granularity: PriceGranularity, } /// Stream the publisher response body through the processing pipeline. @@ -441,6 +448,261 @@ pub fn stream_publisher_body( process_response_streaming(body, output, &borrowed) } +/// Stream publisher body with a "last-chunk hold" for live bid injection. +/// +/// Drives the origin body through the HTML pipeline one chunk at a time, using a +/// one-behind buffer so the last raw origin chunk is held back. When the origin +/// body is exhausted (`read` returns `Ok(0)`): +/// +/// 1. [`collect_dispatched_auction`](AuctionOrchestrator::collect_dispatched_auction) +/// is awaited with the remaining deadline. +/// 2. Winning bids are written to `ad_bids_state`. +/// 3. The held last chunk is fed through the pipeline — `lol_html` fires its +/// `` handler with bids now in state. +/// +/// For non-HTML content types the auction is collected before any body bytes +/// are written (no `` to inject). If `params.dispatched_auction` is +/// `None` the function falls back to the synchronous +/// [`stream_publisher_body`] path. +/// +/// # Errors +/// +/// Returns an error if processing fails mid-stream. Headers are already +/// committed at that point; the caller logs and drops the `StreamingBody`. +pub async fn stream_publisher_body_async( + body: Body, + output: &mut W, + params: &mut OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, +) -> Result<(), Report> { + let Some(dispatched) = params.dispatched_auction.take() else { + // No auction — use the existing sync pipeline unchanged. + return stream_publisher_body(body, output, params, settings, integration_registry); + }; + + let is_html = params.content_type.contains("text/html"); + + if !is_html { + // Non-HTML: collect auction first, then stream. There is no + // to hold, so delaying the entire body until collection is acceptable. + let placeholder = Request::get("https://placeholder.invalid/"); + let result = orchestrator + .collect_dispatched_auction(dispatched, services, &make_collect_context(settings, services, &placeholder)) + .await; + write_bids_to_state(&result.winning_bids, params.price_granularity, ¶ms.ad_bids_state); + return stream_publisher_body(body, output, params, settings, integration_registry); + } + + // HTML: build the processor once and drive it chunk by chunk. + // One-behind buffer: stream chunk N-1 immediately; hold chunk N until origin + // EOF, then await auction and process chunk N (which contains ). + let mut processor = create_html_stream_processor( + ¶ms.origin_host, + ¶ms.request_host, + ¶ms.request_scheme, + settings, + integration_registry, + params.ad_slots_script.as_deref().map(str::to_string), + params.ad_bids_state.clone(), + )?; + + let compression = Compression::from_content_encoding(¶ms.content_encoding); + stream_html_with_auction_hold( + body, + output, + &mut processor, + compression, + AuctionCollectCtx { + dispatched, + price_granularity: params.price_granularity, + ad_bids_state: ¶ms.ad_bids_state, + orchestrator, + services, + settings, + }, + ) + .await +} + +/// Build a minimal [`AuctionContext`] for the mediator call in collection. +/// +/// The `request` field is a short-lived placeholder (providers use it only for +/// header extraction; the placeholder is functionally equivalent to the original +/// since `req` was already consumed by `send_async` before dispatch). +fn make_collect_context<'a>( + settings: &'a Settings, + services: &'a RuntimeServices, + placeholder: &'a Request, +) -> AuctionContext<'a> { + AuctionContext { + settings, + request: placeholder, + client_info: services.client_info(), + timeout_ms: 0, + provider_responses: None, + services, + } +} + +/// Write winning bids from an auction result into the shared `ad_bids_state` lock. +pub(crate) fn write_bids_to_state( + winning_bids: &std::collections::HashMap, + price_granularity: PriceGranularity, + ad_bids_state: &Arc>>, +) { + let bid_map = build_bid_map(winning_bids, price_granularity); + let bids_script = build_bids_script(&bid_map); + *ad_bids_state.write().expect("should write bid state") = Some(bids_script); +} + +/// Bundles the auction-collection dependencies passed through the streaming helpers. +struct AuctionCollectCtx<'a> { + dispatched: DispatchedAuction, + price_granularity: PriceGranularity, + ad_bids_state: &'a Arc>>, + orchestrator: &'a AuctionOrchestrator, + services: &'a RuntimeServices, + settings: &'a Settings, +} + +/// Run the one-behind chunk loop for HTML bodies, collecting the auction before +/// the last chunk so `lol_html`'s `` handler sees live bids. +async fn stream_html_with_auction_hold( + body: Body, + output: &mut W, + processor: &mut P, + compression: Compression, + ctx: AuctionCollectCtx<'_>, +) -> Result<(), Report> { + use brotli::enc::writer::CompressorWriter; + use brotli::enc::BrotliEncoderParams; + use brotli::Decompressor; + use flate2::read::{GzDecoder, ZlibDecoder}; + use flate2::write::{GzEncoder, ZlibEncoder}; + + match compression { + Compression::None => one_behind_loop(body, output, processor, ctx).await, + Compression::Gzip => { + let decoder = GzDecoder::new(body); + let mut encoder = GzEncoder::new(&mut *output, flate2::Compression::default()); + one_behind_loop(decoder, &mut encoder, processor, ctx).await?; + encoder.finish().change_context(TrustedServerError::Proxy { + message: "Failed to finalize gzip encoder".to_string(), + })?; + Ok(()) + } + Compression::Deflate => { + let decoder = ZlibDecoder::new(body); + let mut encoder = ZlibEncoder::new(&mut *output, flate2::Compression::default()); + one_behind_loop(decoder, &mut encoder, processor, ctx).await?; + encoder.finish().change_context(TrustedServerError::Proxy { + message: "Failed to finalize deflate encoder".to_string(), + })?; + Ok(()) + } + Compression::Brotli => { + let decoder = Decompressor::new(body, 4096); + let params = BrotliEncoderParams { + quality: 4, + lgwin: 22, + ..Default::default() + }; + let mut encoder = CompressorWriter::with_params(&mut *output, 4096, ¶ms); + one_behind_loop(decoder, &mut encoder, processor, ctx).await?; + let _ = encoder.into_inner(); + Ok(()) + } + } +} + +/// Core one-behind chunk loop. +/// +/// Reads from `reader`, writing processed output to `writer` for every chunk +/// except the current one (which is held pending). On EOF, the auction is +/// collected, bids written, and the held chunk processed last. +async fn one_behind_loop( + mut reader: R, + writer: &mut W, + processor: &mut P, + ctx: AuctionCollectCtx<'_>, +) -> Result<(), Report> { + let AuctionCollectCtx { dispatched, price_granularity, ad_bids_state, orchestrator, services, settings } = ctx; + const CHUNK_SIZE: usize = 8192; + let mut buffer = vec![0u8; CHUNK_SIZE]; + let mut pending: Vec = Vec::new(); + + loop { + match reader.read(&mut buffer) { + Ok(0) => { + // Origin exhausted — pending holds the last chunk. + // Collect the auction before feeding it to lol_html so that + // the handler sees populated ad_bids_state. + let placeholder = Request::get("https://placeholder.invalid/"); + let collect_ctx = make_collect_context(settings, services, &placeholder); + let result = orchestrator + .collect_dispatched_auction(dispatched, services, &collect_ctx) + .await; + write_bids_to_state(&result.winning_bids, price_granularity, ad_bids_state); + + // Process the held last chunk (not is_last — finalization is separate). + if !pending.is_empty() { + let out = processor.process_chunk(&pending, false).change_context( + TrustedServerError::Proxy { + message: "Failed to process last chunk".to_string(), + }, + )?; + if !out.is_empty() { + writer.write_all(&out).change_context(TrustedServerError::Proxy { + message: "Failed to write last chunk".to_string(), + })?; + } + } + // Signal EOF to lol_html (fires end() which flushes remaining state). + let final_out = processor.process_chunk(&[], true).change_context( + TrustedServerError::Proxy { + message: "Failed to finalize processor".to_string(), + }, + )?; + if !final_out.is_empty() { + writer.write_all(&final_out).change_context(TrustedServerError::Proxy { + message: "Failed to write finalized output".to_string(), + })?; + } + break; + } + Ok(n) => { + // Stream the previously held chunk (it is not the last). + if !pending.is_empty() { + let out = processor.process_chunk(&pending, false).change_context( + TrustedServerError::Proxy { + message: "Failed to process chunk".to_string(), + }, + )?; + if !out.is_empty() { + writer.write_all(&out).change_context(TrustedServerError::Proxy { + message: "Failed to write chunk".to_string(), + })?; + } + } + pending = buffer[..n].to_vec(); + } + Err(e) => { + return Err(Report::new(TrustedServerError::Proxy { + message: format!("Failed to read origin body: {e}"), + })); + } + } + } + + writer.flush().change_context(TrustedServerError::Proxy { + message: "Failed to flush output".to_string(), + })?; + Ok(()) +} + /// Proxies requests to the publisher's origin server. /// /// Returns a [`PublisherResponse`] indicating how the response should be sent: @@ -590,13 +852,22 @@ pub async fn handle_publisher_request( restrict_accept_encoding(&mut req); req.set_header("host", &origin_host); + // Dispatch origin request first. let pending_origin = req.send_async(&backend_name) .change_context(TrustedServerError::Proxy { message: "Failed to dispatch async origin request".to_string(), })?; - let auction_result = if should_run_auction { + // Dispatch SSP bid requests BEFORE awaiting origin — all HTTP is now in-flight + // in Fastly's native layer. WASM yields only for origin (fast, cache-hit path), + // so TTFB ≈ origin latency instead of TTFB ≈ auction timeout. + let price_granularity = settings + .creative_opportunities + .as_ref() + .map(|co| co.price_granularity) + .unwrap_or_default(); + let dispatched_auction = if should_run_auction { let co_config = settings .creative_opportunities .as_ref() @@ -617,35 +888,12 @@ pub async fn handle_publisher_request( provider_responses: None, services, }; - match orchestrator - .run_auction(&auction_request, &auction_context, services) - .await - { - Ok(result) => Some(result), - Err(e) => { - log::warn!("server-side auction failed, proceeding without bids: {e:?}"); - None - } - } + orchestrator.dispatch_auction(&auction_request, &auction_context) } else { None }; - if should_run_auction { - let co_config = settings - .creative_opportunities - .as_ref() - .expect("should be present"); - let empty: std::collections::HashMap = std::collections::HashMap::new(); - let winning_bids = auction_result - .as_ref() - .map(|r| &r.winning_bids) - .unwrap_or(&empty); - let bid_map = build_bid_map(winning_bids, co_config.price_granularity); - let bids_script = build_bids_script(&bid_map); - *ad_bids_state.write().expect("should write bid state") = Some(bids_script); - } - + // Now yield for origin — SSP requests are already racing in Fastly's native layer. let mut response = pending_origin .wait() .change_context(TrustedServerError::Proxy { @@ -754,7 +1002,7 @@ pub async fn handle_publisher_request( Ok(PublisherResponse::Stream { response, body, - params: OwnedProcessResponseParams { + params: Box::new(OwnedProcessResponseParams { content_encoding, origin_host, origin_url: settings.publisher.origin_url.clone(), @@ -763,7 +1011,9 @@ pub async fn handle_publisher_request( content_type, ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), - }, + dispatched_auction, + price_granularity, + }), }) } ResponseRoute::BufferedProcessed => { @@ -1790,6 +2040,9 @@ mod tests { content_type: "text/css".to_string(), ad_slots_script: None, ad_bids_state: Arc::new(RwLock::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; let mut output = Vec::new(); @@ -1833,6 +2086,9 @@ mod tests { content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, ad_bids_state: Arc::new(RwLock::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; let mut output = Vec::new(); @@ -1867,6 +2123,9 @@ mod tests { content_type: "text/html".to_string(), ad_slots_script: None, ad_bids_state: Arc::new(RwLock::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; let bogus_body = Body::from(b"not gzip".to_vec()); @@ -1968,6 +2227,9 @@ mod tests { content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, ad_bids_state: Arc::new(RwLock::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -2020,6 +2282,9 @@ mod tests { content_type: "text/html".to_string(), ad_slots_script: None, ad_bids_state: Arc::new(RwLock::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; let mut output = Vec::new(); diff --git a/trusted-server.toml b/trusted-server.toml index 43e090fea..b1b5a0b03 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -192,6 +192,12 @@ permutive_segments = "permutive" [creative_opportunities] gam_network_id = "88059007" +# FCP is not affected by this value — body content above has already +# streamed and painted before the hold begins. What this caps is the slip on +# DOMContentLoaded and window.load. Worst case: a cache-hit page where origin +# drains in <50 ms but the auction runs to the limit. 500 ms is the recommended +# default; raise only if your SSPs need more headroom and your analytics confirm +# the DCL slip is acceptable. auction_timeout_ms = 1500 price_granularity = "dense" From a2d08e78ea7c4bc3fd73c4259d3cb4b66f37a153 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 10 May 2026 16:13:47 +0530 Subject: [PATCH 041/395] Fix clippy explicit-auto-deref in stream_publisher_body_async call --- crates/trusted-server-adapter-fastly/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 895299f54..94f095193 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -262,7 +262,7 @@ async fn route_request( let stream_result = stream_publisher_body_async( body, &mut streaming_body, - &mut *params, + &mut params, settings, integration_registry, orchestrator, From b03af6b1f94b4bb34b8a59db8b28b69a747287ca Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 10 May 2026 16:34:20 +0530 Subject: [PATCH 042/395] =?UTF-8?q?Fix=20Cache-Control=20headers=20applied?= =?UTF-8?q?=20only=20when=20slots=20matched=20=E2=80=94=20apply=20to=20all?= =?UTF-8?q?=20HTML=20responses=20per=20spec=20=C2=A74.7=20+=20=C2=A78?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/trusted-server-core/src/publisher.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4a39c9623..dc93af768 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -915,7 +915,13 @@ pub async fn handle_publisher_request( None }; - if ad_slots_script.is_some() { + // §4.7: assembled HTML responses must never be shared-cached — per-user bid data + // travels inline. Apply regardless of slot match or auction outcome (§8). + let origin_content_type = response + .get_header(header::CONTENT_TYPE) + .and_then(|h| h.to_str().ok()) + .unwrap_or_default(); + if origin_content_type.contains("text/html") { response.set_header(header::CACHE_CONTROL, "private, max-age=0"); response.remove_header("surrogate-control"); response.remove_header("fastly-surrogate-control"); From 349dcdcb2689d039085e1267f0011f8e94f00b2a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 10 May 2026 16:45:54 +0530 Subject: [PATCH 043/395] cargo fmt --- .../src/auction/orchestrator.rs | 111 +++++++++++++----- crates/trusted-server-core/src/publisher.rs | 50 +++++--- 2 files changed, 114 insertions(+), 47 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 953ed6a04..820031050 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -637,7 +637,10 @@ impl AuctionOrchestrator { }; if !provider.is_enabled() { - log::debug!("Provider '{}' is disabled, skipping", provider.provider_name()); + log::debug!( + "Provider '{}' is disabled, skipping", + provider.provider_name() + ); continue; } @@ -656,7 +659,10 @@ impl AuctionOrchestrator { let backend_name = match provider.backend_name(effective_timeout) { Some(name) => name, None => { - log::warn!("Provider '{}' has no backend_name, skipping", provider.provider_name()); + log::warn!( + "Provider '{}' has no backend_name, skipping", + provider.provider_name() + ); continue; } }; @@ -681,7 +687,11 @@ impl AuctionOrchestrator { ); backend_to_provider.insert( backend_name.clone(), - (provider.provider_name().to_string(), start_time, Arc::clone(provider)), + ( + provider.provider_name().to_string(), + start_time, + Arc::clone(provider), + ), ); pending_requests .push(PlatformPendingRequest::new(pending).with_backend_name(backend_name)); @@ -777,28 +787,45 @@ impl AuctionOrchestrator { { let response_time_ms = start_time.elapsed().as_millis() as u64; match platform_response_to_fastly(platform_response) { - Ok(response) => match provider.parse_response(response, response_time_ms) { - Ok(auction_response) => { - log::info!( - "Provider '{}' returned {} bids ({}ms)", - auction_response.provider, - auction_response.bids.len(), - auction_response.response_time_ms - ); - responses.push(auction_response); - } - Err(e) => { - log::warn!("Provider '{}' parse failed: {:?}", provider_name, e); - responses.push(AuctionResponse::error(&provider_name, response_time_ms)); + Ok(response) => { + match provider.parse_response(response, response_time_ms) { + Ok(auction_response) => { + log::info!( + "Provider '{}' returned {} bids ({}ms)", + auction_response.provider, + auction_response.bids.len(), + auction_response.response_time_ms + ); + responses.push(auction_response); + } + Err(e) => { + log::warn!( + "Provider '{}' parse failed: {:?}", + provider_name, + e + ); + responses.push(AuctionResponse::error( + &provider_name, + response_time_ms, + )); + } } - }, + } Err(e) => { - log::warn!("Provider '{}' unsupported body: {:?}", provider_name, e); - responses.push(AuctionResponse::error(&provider_name, response_time_ms)); + log::warn!( + "Provider '{}' unsupported body: {:?}", + provider_name, + e + ); + responses + .push(AuctionResponse::error(&provider_name, response_time_ms)); } } } else { - log::warn!("Received response from unknown backend '{}', ignoring", backend_name); + log::warn!( + "Received response from unknown backend '{}', ignoring", + backend_name + ); } } Err(e) => { @@ -847,18 +874,27 @@ impl AuctionOrchestrator { .wait(PlatformPendingRequest::new(pending)) .await; match platform_resp.change_context(TrustedServerError::Auction { - message: format!("Mediator {} request failed", mediator.provider_name()), + message: format!( + "Mediator {} request failed", + mediator.provider_name() + ), }) { Ok(platform_resp) => { match platform_response_to_fastly(platform_resp).change_context( TrustedServerError::Auction { - message: format!("Mediator {} unsupported body", mediator.provider_name()), + message: format!( + "Mediator {} unsupported body", + mediator.provider_name() + ), }, ) { Ok(response) => { - let response_time_ms = - remaining_ms as u64 - remaining_budget_ms(auction_start, timeout_ms) as u64; - match mediator.parse_response(response, response_time_ms) { + let response_time_ms = remaining_ms as u64 + - remaining_budget_ms(auction_start, timeout_ms) + as u64; + match mediator + .parse_response(response, response_time_ms) + { Ok(mediator_resp) => { let winning = mediator_resp .bids @@ -876,19 +912,30 @@ impl AuctionOrchestrator { } }) .collect(); - let winning = self.apply_floor_prices(winning, &floor_prices); + let winning = self + .apply_floor_prices(winning, &floor_prices); (Some(mediator_resp), winning) } Err(e) => { - log::warn!("Mediator '{}' parse failed: {:?}", mediator.provider_name(), e); - let winning = self.select_winning_bids(&responses, &floor_prices); + log::warn!( + "Mediator '{}' parse failed: {:?}", + mediator.provider_name(), + e + ); + let winning = self.select_winning_bids( + &responses, + &floor_prices, + ); (None, winning) } } } Err(e) => { log::warn!("Mediator body error: {:?}", e); - (None, self.select_winning_bids(&responses, &floor_prices)) + ( + None, + self.select_winning_bids(&responses, &floor_prices), + ) } } } @@ -899,7 +946,11 @@ impl AuctionOrchestrator { } } Err(e) => { - log::warn!("Mediator '{}' failed to dispatch: {:?}", mediator.provider_name(), e); + log::warn!( + "Mediator '{}' failed to dispatch: {:?}", + mediator.provider_name(), + e + ); (None, self.select_winning_bids(&responses, &floor_prices)) } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index dc93af768..3a10e85f1 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -490,9 +490,17 @@ pub async fn stream_publisher_body_async( // to hold, so delaying the entire body until collection is acceptable. let placeholder = Request::get("https://placeholder.invalid/"); let result = orchestrator - .collect_dispatched_auction(dispatched, services, &make_collect_context(settings, services, &placeholder)) + .collect_dispatched_auction( + dispatched, + services, + &make_collect_context(settings, services, &placeholder), + ) .await; - write_bids_to_state(&result.winning_bids, params.price_granularity, ¶ms.ad_bids_state); + write_bids_to_state( + &result.winning_bids, + params.price_granularity, + ¶ms.ad_bids_state, + ); return stream_publisher_body(body, output, params, settings, integration_registry); } @@ -629,7 +637,14 @@ async fn one_behind_loop( processor: &mut P, ctx: AuctionCollectCtx<'_>, ) -> Result<(), Report> { - let AuctionCollectCtx { dispatched, price_granularity, ad_bids_state, orchestrator, services, settings } = ctx; + let AuctionCollectCtx { + dispatched, + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + } = ctx; const CHUNK_SIZE: usize = 8192; let mut buffer = vec![0u8; CHUNK_SIZE]; let mut pending: Vec = Vec::new(); @@ -655,9 +670,11 @@ async fn one_behind_loop( }, )?; if !out.is_empty() { - writer.write_all(&out).change_context(TrustedServerError::Proxy { - message: "Failed to write last chunk".to_string(), - })?; + writer + .write_all(&out) + .change_context(TrustedServerError::Proxy { + message: "Failed to write last chunk".to_string(), + })?; } } // Signal EOF to lol_html (fires end() which flushes remaining state). @@ -667,9 +684,11 @@ async fn one_behind_loop( }, )?; if !final_out.is_empty() { - writer.write_all(&final_out).change_context(TrustedServerError::Proxy { - message: "Failed to write finalized output".to_string(), - })?; + writer + .write_all(&final_out) + .change_context(TrustedServerError::Proxy { + message: "Failed to write finalized output".to_string(), + })?; } break; } @@ -682,9 +701,11 @@ async fn one_behind_loop( }, )?; if !out.is_empty() { - writer.write_all(&out).change_context(TrustedServerError::Proxy { - message: "Failed to write chunk".to_string(), - })?; + writer + .write_all(&out) + .change_context(TrustedServerError::Proxy { + message: "Failed to write chunk".to_string(), + })?; } } pending = buffer[..n].to_vec(); @@ -2048,7 +2069,6 @@ mod tests { ad_bids_state: Arc::new(RwLock::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - }; let mut output = Vec::new(); @@ -2094,7 +2114,6 @@ mod tests { ad_bids_state: Arc::new(RwLock::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - }; let mut output = Vec::new(); @@ -2131,7 +2150,6 @@ mod tests { ad_bids_state: Arc::new(RwLock::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - }; let bogus_body = Body::from(b"not gzip".to_vec()); @@ -2235,7 +2253,6 @@ mod tests { ad_bids_state: Arc::new(RwLock::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -2290,7 +2307,6 @@ mod tests { ad_bids_state: Arc::new(RwLock::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - }; let mut output = Vec::new(); From 78885f9c1f53007be16c1c18e480487c6e6a9fc3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 10 May 2026 19:54:52 +0530 Subject: [PATCH 044/395] =?UTF-8?q?Fix=20auction=20consent=20gate=20blocki?= =?UTF-8?q?ng=20non-GDPR=20regions=20=E2=80=94=20only=20require=20TCF=20Pu?= =?UTF-8?q?rpose=201=20when=20gdpr=5Fapplies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/trusted-server-core/src/publisher.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 3a10e85f1..81637ae53 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -853,10 +853,13 @@ pub async fn handle_publisher_request( Vec::new() }; - let consent_allows_auction = consent_context - .tcf - .as_ref() - .is_some_and(|tcf| tcf.has_purpose_consent(1)); + // Non-GDPR regions (US, etc.) have no TCF string — auction is freely allowed. + // GDPR regions require TCF Purpose 1 (storage/access) before firing. + let consent_allows_auction = !consent_context.gdpr_applies + || consent_context + .tcf + .as_ref() + .is_some_and(|tcf| tcf.has_purpose_consent(1)); let should_run_auction = is_get && !is_prefetch && !is_bot && !matched_slots.is_empty() && consent_allows_auction; @@ -1324,10 +1327,11 @@ pub async fn handle_page_bids( .map(|_| services.kv_store()), }); - let consent_allows_auction = consent_context - .tcf - .as_ref() - .is_some_and(|tcf| tcf.has_purpose_consent(1)); + let consent_allows_auction = !consent_context.gdpr_applies + || consent_context + .tcf + .as_ref() + .is_some_and(|tcf| tcf.has_purpose_consent(1)); let winning_bids = if !matched_slots.is_empty() && consent_allows_auction { let mut auction_request = build_auction_request( From 3783e68104258e71b8325820ac1c34026b928fc0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 10 May 2026 20:52:05 +0530 Subject: [PATCH 045/395] =?UTF-8?q?Fix=20SSP=20requests=20using=20placehol?= =?UTF-8?q?der=20headers=20=E2=80=94=20pass=20real=20request=20to=20dispat?= =?UTF-8?q?ch=5Fauction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dispatch_auction was building AuctionContext with a placeholder Request (GET https://placeholder.invalid/) that carried no headers. Prebid's request_bids copies User-Agent, x-forwarded-for, Referer, Accept-Language, and cookies from context.request before sending to Prebid Server, so SSPs received stripped requests and returned empty bids. Fix: dispatch SSP requests before req.send_async(), using the original request directly as AuctionContext.request. DispatchedAuction holds no lifetime reference to Request, so the borrow ends at return and req can be modified (restrict_accept_encoding, Host header) and sent to origin immediately after. --- crates/trusted-server-core/src/publisher.rs | 36 +++++++++++---------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 81637ae53..f5caa5710 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -872,25 +872,16 @@ pub async fn handle_publisher_request( let ad_bids_state: Arc>> = Arc::new(RwLock::new(None)); - // Only advertise encodings the rewrite pipeline can decode and re-encode. - restrict_accept_encoding(&mut req); - req.set_header("host", &origin_host); - - // Dispatch origin request first. - let pending_origin = - req.send_async(&backend_name) - .change_context(TrustedServerError::Proxy { - message: "Failed to dispatch async origin request".to_string(), - })?; - - // Dispatch SSP bid requests BEFORE awaiting origin — all HTTP is now in-flight - // in Fastly's native layer. WASM yields only for origin (fast, cache-hit path), - // so TTFB ≈ origin latency instead of TTFB ≈ auction timeout. let price_granularity = settings .creative_opportunities .as_ref() .map(|co| co.price_granularity) .unwrap_or_default(); + + // Dispatch SSP bid requests while req still has the original client headers + // (User-Agent, x-forwarded-for, cookies, etc.). The borrow ends when + // dispatch_auction returns — DispatchedAuction holds no lifetime — so req + // can be mutated and sent to origin immediately after. let dispatched_auction = if should_run_auction { let co_config = settings .creative_opportunities @@ -903,10 +894,9 @@ pub async fn handle_publisher_request( &request_info, co_config, ); - let placeholder_req = fastly::Request::get("https://placeholder.invalid/"); let auction_context = AuctionContext { settings, - request: &placeholder_req, + request: &req, client_info: services.client_info(), timeout_ms: auction_timeout_ms, provider_responses: None, @@ -917,7 +907,19 @@ pub async fn handle_publisher_request( None }; - // Now yield for origin — SSP requests are already racing in Fastly's native layer. + // Only advertise encodings the rewrite pipeline can decode and re-encode. + restrict_accept_encoding(&mut req); + req.set_header("host", &origin_host); + + // Dispatch origin — SSP requests are already racing in Fastly's native layer. + // TTFB ≈ origin latency instead of TTFB ≈ auction timeout. + let pending_origin = + req.send_async(&backend_name) + .change_context(TrustedServerError::Proxy { + message: "Failed to dispatch async origin request".to_string(), + })?; + + // Now yield for origin. let mut response = pending_origin .wait() .change_context(TrustedServerError::Proxy { From 0c9465206f4e6b3ad865277dae63378090831a79 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 12:36:25 +0530 Subject: [PATCH 046/395] Fix async auction collect abandoning SSP bids when origin is slow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In collect_dispatched_auction, the select loop checked `auction_start.elapsed() >= deadline` after each SSP response and broke early if the 1500ms budget had elapsed. When origin TTFB + body download exceeded the auction budget, the check fired after collecting the first SSP response, abandoning the second SSP's already-buffered response. This left responses with only one (possibly errored) SSP, causing remaining_ms == 0 which skipped the mediator, and select_winning_bids on the partial set returned zero bids. The deadline break is wrong in this context: SSP HTTP connections are already bounded by the backend first_byte_timeout set at dispatch time (1000ms per provider). By the time collect is called at origin EOF, all SSPs have either responded or been errored by Fastly's host. The select() calls drain instantly — no WASM-level deadline enforcement is needed or safe. Also add info-level log statements at dispatch, collect, and write_bids_to_state to make the auction pipeline observable without requiring a dashboard. --- .../src/auction/orchestrator.rs | 10 --------- crates/trusted-server-core/src/publisher.rs | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 820031050..867bdf3a7 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -751,8 +751,6 @@ impl AuctionOrchestrator { request, } = dispatched; - let deadline = Duration::from_millis(u64::from(timeout_ms)); - log::info!( "Collecting {} in-flight SSP responses (timeout: {}ms remaining: {}ms)", pending_requests.len(), @@ -833,14 +831,6 @@ impl AuctionOrchestrator { } } - if auction_start.elapsed() >= deadline && !remaining.is_empty() { - log::warn!( - "Auction timeout ({}ms) reached, dropping {} remaining request(s)", - timeout_ms, - remaining.len() - ); - break; - } } let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index f5caa5710..5284e20f3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -561,6 +561,15 @@ pub(crate) fn write_bids_to_state( price_granularity: PriceGranularity, ad_bids_state: &Arc>>, ) { + log::info!( + "write_bids_to_state: {} winning bid(s): [{}]", + winning_bids.len(), + winning_bids + .keys() + .cloned() + .collect::>() + .join(", ") + ); let bid_map = build_bid_map(winning_bids, price_granularity); let bids_script = build_bids_script(&bid_map); *ad_bids_state.write().expect("should write bid state") = Some(bids_script); @@ -655,11 +664,16 @@ async fn one_behind_loop( // Origin exhausted — pending holds the last chunk. // Collect the auction before feeding it to lol_html so that // the handler sees populated ad_bids_state. + log::info!("one_behind_loop: EOF — collecting dispatched auction"); let placeholder = Request::get("https://placeholder.invalid/"); let collect_ctx = make_collect_context(settings, services, &placeholder); let result = orchestrator .collect_dispatched_auction(dispatched, services, &collect_ctx) .await; + log::info!( + "one_behind_loop: collect complete — {} winning bid(s)", + result.winning_bids.len() + ); write_bids_to_state(&result.winning_bids, price_granularity, ad_bids_state); // Process the held last chunk (not is_last — finalization is separate). @@ -906,6 +920,14 @@ pub async fn handle_publisher_request( } else { None }; + log::info!( + "dispatch_auction: {}", + if dispatched_auction.is_some() { + "Some — auction running async" + } else { + "None — falling back to sync or skipped" + } + ); // Only advertise encodings the rewrite pipeline can decode and re-encode. restrict_accept_encoding(&mut req); From f172f4477c60ae0fc0e0d68b0d9f969652dc092f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 13:15:23 +0530 Subject: [PATCH 047/395] Cargo fmt --- crates/trusted-server-core/src/auction/orchestrator.rs | 1 - crates/trusted-server-core/src/publisher.rs | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 867bdf3a7..e9d8fa198 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -830,7 +830,6 @@ impl AuctionOrchestrator { log::warn!("A provider request failed during collection: {:?}", e); } } - } let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 5284e20f3..3a9eb0895 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -564,11 +564,7 @@ pub(crate) fn write_bids_to_state( log::info!( "write_bids_to_state: {} winning bid(s): [{}]", winning_bids.len(), - winning_bids - .keys() - .cloned() - .collect::>() - .join(", ") + winning_bids.keys().cloned().collect::>().join(", ") ); let bid_map = build_bid_map(winning_bids, price_granularity); let bids_script = build_bids_script(&bid_map); From 14cd493ee907fbf2cce5030062bfed4ccac41043 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 13:32:28 +0530 Subject: [PATCH 048/395] Fix mediator always skipped when origin body exceeds SSP auction budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In collect_dispatched_auction, the mediator was skipped when remaining_budget_ms(auction_start, timeout_ms) == 0. In the async-dispatch path, auction_start is set before pending_origin.wait(), so elapsed time includes the full origin TTFB and body download. For heavy SSR pages (autoblog), this exceeds the 1500ms SSP budget, making remaining_ms == 0 at every collection and causing the mediator to be permanently skipped. The mediator (adserver_mock) is the primary bid source — SSPs alone return no bids. Skipping it means window.__ts_bids == {} on every full page load, while handle_page_bids (which uses the sequential run_auction path) works correctly because it measures remaining time from after SSP collection. Fix: give the mediator its own configured timeout (mediator.timeout_ms()) instead of the exhausted SSP budget. This mirrors how run_parallel_mediation works: the mediator's deadline is independent of SSP round-trip time. Side effect: mediator backend name is now stable (always t1000 for adserver_mock) rather than varying per request with remaining_ms. --- .../src/auction/orchestrator.rs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index e9d8fa198..892ff9ebb 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -835,24 +835,25 @@ impl AuctionOrchestrator { let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { match self.providers.get(mediator_name.as_str()) { Some(mediator) => { - let remaining_ms = remaining_budget_ms(auction_start, timeout_ms); - if remaining_ms == 0 { - log::warn!("Auction timeout exhausted during bidding — skipping mediator"); - let winning = self.select_winning_bids(&responses, &floor_prices); - return OrchestrationResult { - provider_responses: responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), - }; - } + // Use the mediator's own configured timeout, not the remaining SSP + // budget. In the async-dispatch path, SSPs race against origin, so + // auction_start.elapsed() can exceed the SSP budget by the time the + // origin body finishes streaming. Skipping the mediator in that case + // would discard all bids — the mediator is the primary bid source. + let mediator_timeout = mediator.timeout_ms(); + let mediator_start = Instant::now(); + log::info!( + "Running mediator '{}' with {}ms budget (SSP budget remaining: {}ms)", + mediator.provider_name(), + mediator_timeout, + remaining_budget_ms(auction_start, timeout_ms), + ); let placeholder = fastly::Request::get("https://placeholder.invalid/"); let mediator_context = AuctionContext { settings: context.settings, request: &placeholder, client_info: context.client_info, - timeout_ms: remaining_ms, + timeout_ms: mediator_timeout, provider_responses: Some(&responses), services: context.services, }; @@ -878,9 +879,8 @@ impl AuctionOrchestrator { }, ) { Ok(response) => { - let response_time_ms = remaining_ms as u64 - - remaining_budget_ms(auction_start, timeout_ms) - as u64; + let response_time_ms = + mediator_start.elapsed().as_millis() as u64; match mediator .parse_response(response, response_time_ms) { From 6210ebbf1560558813c7d30865b032a5a7962649 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 14:04:28 +0530 Subject: [PATCH 049/395] Cargo fmt --- crates/trusted-server-core/src/publisher.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 6fd9b0d6a..30abc5326 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1332,7 +1332,8 @@ pub async fn handle_page_bids( .collect(); let http_req = compat::from_fastly_headers_ref(&req); - let request_info = crate::http_util::RequestInfo::from_request(&http_req, &services.client_info); + let request_info = + crate::http_util::RequestInfo::from_request(&http_req, &services.client_info); let cookie_jar = handle_request_cookies(&http_req)?; let ec_id = get_or_generate_ec_id_from_http_request(settings, services, &http_req)?; let geo = services From 3cdf9952f54fcfe4c4c7cc87f8064aa1af8aa721 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 14:44:58 +0530 Subject: [PATCH 050/395] Adding debug info for auction --- crates/trusted-server-core/src/publisher.rs | 44 +++++++++++++++------ crates/trusted-server-core/src/settings.rs | 6 +++ trusted-server.toml | 6 ++- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 30abc5326..ba594d808 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -677,6 +677,29 @@ async fn one_behind_loop( ); write_bids_to_state(&result.winning_bids, price_granularity, ad_bids_state); + if settings.debug.auction_html_comment { + let ssp_count = result.provider_responses.len(); + let mediator_info = match &result.mediator_response { + Some(r) => format!("ok({}_bids)", r.bids.len()), + None => "none".to_string(), + }; + let debug_comment = format!( + "", + result.winning_bids.len() + ); + let mut state = ad_bids_state + .write() + .expect("should write bid state for debug"); + match &mut *state { + Some(script) => { + *script = format!("{debug_comment}\n{script}"); + } + None => { + *state = Some(debug_comment); + } + } + } + // Process the held last chunk (not is_last — finalization is separate). if !pending.is_empty() { let out = processor.process_chunk(&pending, false).change_context( @@ -909,6 +932,7 @@ pub async fn handle_publisher_request( &ec_id, &consent_context, &request_info, + &request_path, co_config, ); let auction_context = AuctionContext { @@ -1109,18 +1133,23 @@ pub(crate) fn build_auction_request( ec_id: &str, consent_context: &crate::consent::ConsentContext, request_info: &crate::http_util::RequestInfo, + request_path: &str, co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, ) -> AuctionRequest { let slots = matched_slots .iter() .map(|s| s.to_ad_slot(&co_config.gam_network_id)) .collect(); + let page_url = format!( + "{}://{}{}", + request_info.scheme, request_info.host, request_path + ); AuctionRequest { id: format!("ts-{}", ec_id), slots, publisher: PublisherInfo { domain: request_info.host.clone(), - page_url: None, + page_url: Some(page_url.clone()), }, user: UserInfo { id: ec_id.to_string(), @@ -1130,7 +1159,7 @@ pub(crate) fn build_auction_request( device: None, site: Some(SiteInfo { domain: request_info.host.clone(), - page: String::new(), + page: page_url, }), context: std::collections::HashMap::new(), } @@ -1363,21 +1392,14 @@ pub async fn handle_page_bids( .is_some_and(|tcf| tcf.has_purpose_consent(1)); let winning_bids = if !matched_slots.is_empty() && consent_allows_auction { - let mut auction_request = build_auction_request( + let auction_request = build_auction_request( &matched_slots, &ec_id, &consent_context, &request_info, + &path_param, co_config, ); - let page_url = format!( - "{}://{}{}", - request_info.scheme, request_info.host, path_param - ); - auction_request.publisher.page_url = Some(page_url.clone()); - if let Some(ref mut site) = auction_request.site { - site.page = page_url; - } let timeout_ms = co_config .auction_timeout_ms .unwrap_or(settings.auction.timeout_ms); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index e09c50e2d..386f0d54b 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -410,6 +410,12 @@ pub struct DebugConfig { /// Fastly-observed TLS details that browser JS cannot normally read. #[serde(default)] pub ja4_endpoint_enabled: bool, + + /// Inject a `` HTML comment before `` showing + /// auction pipeline stats (SSP count, mediator status, winning bid count). + /// Never enable in production — visible in page source. + #[serde(default)] + pub auction_html_comment: bool, } #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] diff --git a/trusted-server.toml b/trusted-server.toml index 60876389f..a71abfdd7 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -208,7 +208,11 @@ endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" timeout_ms = 1000 # Debug configuration (all flags default to false — do not enable in production) -# [debug] +# TODO: remove [debug] block before merging to main +[debug] +# Inject before . +# Visible in page source. Disable after investigation. +auction_html_comment = true # Enable the JA4/TLS fingerprint debug endpoint at GET /_ts/debug/ja4. # Returns a plain-text response with the following fields (Fastly-observed values): # ja4 — JA4 TLS client fingerprint From e35c593f3b26f64ee148ed39a698b19c23b594db Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 15:30:15 +0530 Subject: [PATCH 051/395] Fix auction bids missing on Next.js buffered HTML path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `classify_response_route` returns `BufferedProcessed` when HTML has post-processors registered (e.g. the Next.js integration registers one via `with_html_post_processor`). Unlike the `Stream` path, which drives `one_behind_loop` to collect the dispatched auction at origin EOF, the `BufferedProcessed` branch previously discarded `dispatched_auction` entirely — so `ad_bids_state` stayed `None` and lol_html injected the fallback `window.__ts_bids = {}` instead of real bids. Fix: collect the in-flight dispatched auction in the `BufferedProcessed` branch before calling `process_response_streaming`, using the same `collect_dispatched_auction` + `write_bids_to_state` pattern that the stream path uses. The `debug.auction_html_comment` injection is mirrored here as well so the comment appears in both code paths when enabled. --- crates/trusted-server-core/src/publisher.rs | 43 +++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index ba594d808..0178d56d1 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1103,6 +1103,49 @@ pub async fn handle_publisher_request( content_type, content_encoding, request_host, origin_host ); + // Collect any in-flight auction before processing buffered HTML. + // BufferedProcessed is taken when HTML has post-processors (e.g. Next.js rewriters). + // Unlike the Stream path, the body is fully buffered first — collect auction + // now so bids are available when the handler fires. + if let Some(dispatched) = dispatched_auction { + let placeholder = fastly::Request::get("https://placeholder.invalid/"); + let result = orchestrator + .collect_dispatched_auction( + dispatched, + services, + &make_collect_context(settings, services, &placeholder), + ) + .await; + log::info!( + "BufferedProcessed: auction collected — {} winning bid(s)", + result.winning_bids.len() + ); + write_bids_to_state(&result.winning_bids, price_granularity, &ad_bids_state); + + if settings.debug.auction_html_comment { + let ssp_count = result.provider_responses.len(); + let mediator_info = match &result.mediator_response { + Some(r) => format!("ok({}_bids)", r.bids.len()), + None => "none".to_string(), + }; + let debug_comment = format!( + "", + result.winning_bids.len() + ); + let mut state = ad_bids_state + .write() + .expect("should write bid state for debug"); + match &mut *state { + Some(script) => { + *script = format!("{debug_comment}\n{script}"); + } + None => { + *state = Some(debug_comment); + } + } + } + } + let body = response.take_body(); let params = ProcessResponseParams { content_encoding: &content_encoding, From 3dac7760f6361d022e831137b37314084d3687b9 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 15:57:54 +0530 Subject: [PATCH 052/395] Add path label and auction time to debug HTML comment --- crates/trusted-server-core/src/publisher.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 0178d56d1..21399ef74 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -684,8 +684,9 @@ async fn one_behind_loop( None => "none".to_string(), }; let debug_comment = format!( - "", - result.winning_bids.len() + "", + result.winning_bids.len(), + result.total_time_ms, ); let mut state = ad_bids_state .write() @@ -1129,8 +1130,9 @@ pub async fn handle_publisher_request( None => "none".to_string(), }; let debug_comment = format!( - "", - result.winning_bids.len() + "", + result.winning_bids.len(), + result.total_time_ms, ); let mut state = ad_bids_state .write() From 65c0ad3090427a84e3655d81c01ff13c7079472f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 19:53:15 +0530 Subject: [PATCH 053/395] Fix XSS in script injection and cap mediator at A_deadline html_escape_for_script now unicode-escapes <, >, & and U+2028/2029 in addition to \ and ". These characters allow a crafted bid value to break out of the ` injection breaking out of the script context +/// - U+2028, U+2029 — line/paragraph separators that are valid JSON but terminate +/// a JS string literal in some parsers +/// +/// All substitutions use `\uXXXX` form, which is valid inside both JSON strings +/// and JS string literals. The result is always safe to write as `JSON.parse("…")`. fn html_escape_for_script(s: &str) -> String { - s.replace('\\', "\\\\").replace('"', "\\\"") + s.replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('<', "\\u003C") + .replace('>', "\\u003E") + .replace('&', "\\u0026") + .replace('\u{2028}', "\\u2028") + .replace('\u{2029}', "\\u2029") } /// Build a price-bucketed bid map from winning bids. @@ -2633,6 +2645,26 @@ mod tests { "both\\\\\\\"mixed", "should escape both backslashes and quotes" ); + assert_eq!( + html_escape_for_script(""), + "\\u003Cscript\\u003Ealert(1)\\u003C/script\\u003E", + "should unicode-escape angle brackets to prevent script injection" + ); + assert_eq!( + html_escape_for_script("a&b"), + "a\\u0026b", + "should unicode-escape ampersand" + ); + assert_eq!( + html_escape_for_script("line\u{2028}sep"), + "line\\u2028sep", + "should unicode-escape U+2028 line separator" + ); + assert_eq!( + html_escape_for_script("para\u{2029}sep"), + "para\\u2029sep", + "should unicode-escape U+2029 paragraph separator" + ); } } } From 0f67a8d9d24368513a7ba68a9fa8e7cf3df20f9f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 13 May 2026 14:36:58 +0530 Subject: [PATCH 054/395] Added footer slot id --- creative-opportunities.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/creative-opportunities.toml b/creative-opportunities.toml index 3cd27f2b1..95ea849a5 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -38,3 +38,22 @@ slot_id = "aps-slot-homepage-header" [slot.providers.pbs.bidders] mocktioneer = { bid = 2.00 } criteo = { networkId = 123456, pubid = "123456" } + +[[slot]] +id = "homepage_footer_ad" +gam_unit_path = "/88059007/autoblog/homepage" +div_id = "ad-fixed_bottom-0-_R_klubtak5lb_" +page_patterns = ["/"] +formats = [{ width = 728, height = 90 }] +floor_price = 0.50 + +[slot.targeting] +pos = "btf" +zone = "fixedBottom" + +[slot.providers.aps] +slot_id = "aps-slot-homepage-footer" + +[slot.providers.pbs.bidders] +mocktioneer = { bid = 1.50 } +criteo = { networkId = 123456, pubid = "123456" } From a7e87512c3bfe7b1fe0296a45e624edfb2498f0a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 15:37:11 +0530 Subject: [PATCH 055/395] Fix page-bids auction context, protect Cache-Control from operator override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass the real incoming request to AuctionContext in handle_page_bids instead of a placeholder — SSPs now receive browser UA, referer, and cookies on SPA navigation bids. Guard Cache-Control in finalize_response so operator response_headers cannot overwrite the private/no-store directives set for per-user HTML and page-bids responses. Disable auction_html_comment debug flag in trusted-server.toml. --- crates/trusted-server-adapter-fastly/src/main.rs | 10 ++++++++++ crates/trusted-server-core/src/publisher.rs | 3 +-- trusted-server.toml | 3 +-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index fc66fcfdc..24c447d3d 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -404,6 +404,16 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: } for (key, value) in &settings.response_headers { + // Never overwrite a privacy-critical Cache-Control header (private, no-store, etc.) + // that was set for per-user responses (HTML or page-bids). + if **key == header::CACHE_CONTROL + && response + .get_header(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("private")) + { + continue; + } response.set_header(key, value); } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 30955f913..5ab8d1aef 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1460,10 +1460,9 @@ pub async fn handle_page_bids( let timeout_ms = co_config .auction_timeout_ms .unwrap_or(settings.auction.timeout_ms); - let placeholder_req = fastly::Request::get("https://placeholder.invalid/"); let auction_context = AuctionContext { settings, - request: &placeholder_req, + request: &req, client_info: services.client_info(), timeout_ms, provider_responses: None, diff --git a/trusted-server.toml b/trusted-server.toml index a71abfdd7..899c8c895 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -208,11 +208,10 @@ endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" timeout_ms = 1000 # Debug configuration (all flags default to false — do not enable in production) -# TODO: remove [debug] block before merging to main [debug] # Inject before . # Visible in page source. Disable after investigation. -auction_html_comment = true +# auction_html_comment = true # Enable the JA4/TLS fingerprint debug endpoint at GET /_ts/debug/ja4. # Returns a plain-text response with the following fields (Fastly-observed values): # ja4 — JA4 TLS client fingerprint From 401136378c6026aa445366b8c0225f132e90ab5a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 16:14:24 +0530 Subject: [PATCH 056/395] Restore nurl/burl/ad_id through adserver_mock mediation path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mock mediator endpoint does not echo nurl/burl/ad_id back in its response. Build a bid index in request_bids keyed by (provider, slot_id, bidder) — where bidder is recovered from the echoed crid field — and restore the fields in parse_mediation_response from the original SSP bids. Fixes the spec requirement: both nurl and burl must travel in __ts_bids for client-side sendBeacon firing on slotRenderEnded (§4.5). --- .../src/integrations/adserver_mock.rs | 81 +++++++++++++++---- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 3a42ec2a0..c8d0ca7b5 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -10,7 +10,7 @@ use fastly::Request; use serde::{Deserialize, Serialize}; use serde_json::{json, Value as Json}; use std::collections::{BTreeMap, HashMap}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::Duration; use validator::Validate; @@ -88,16 +88,28 @@ impl IntegrationConfig for AdServerMockConfig { // Provider // ============================================================================ +/// Lookup index built from original SSP bids during `request_bids`, consumed +/// during `parse_response` to restore `nurl`/`burl`/`ad_id` that the mock +/// mediator endpoint does not echo back. +/// +/// Keyed by `(provider_name, slot_id, bidder_name)`. +type BidIndex = HashMap<(String, String, String), Bid>; + /// Mock ad server mediator provider. pub struct AdServerMockProvider { config: AdServerMockConfig, + /// Bridges SSP bid metadata (nurl/burl/ad_id) from request_bids to parse_response. + bid_index: Mutex>, } impl AdServerMockProvider { /// Create a new mock ad server provider. #[must_use] pub fn new(config: AdServerMockConfig) -> Self { - Self { config } + Self { + config, + bid_index: Mutex::new(None), + } } /// Build the mediation endpoint URL, appending context values as query @@ -212,8 +224,17 @@ impl AdServerMockProvider { /// Parse `OpenRTB` response from mediation endpoint. /// Mediation returns decoded prices for all bids (including APS bids that were encoded). - fn parse_mediation_response(&self, json: &Json, response_time_ms: u64) -> AuctionResponse { - // Parse OpenRTB response + /// + /// `bid_index` is the SSP-bid lookup built in `request_bids`. The mock mediator + /// does not echo `nurl`/`burl`/`ad_id` back, so they are restored from the index + /// using `(seat, impid, bidder)` where bidder is recovered from the echoed `crid` + /// field (`"{bidder}-creative"` format set during request construction). + fn parse_mediation_response( + &self, + json: &Json, + response_time_ms: u64, + bid_index: &BidIndex, + ) -> AuctionResponse { let empty_array = vec![]; let seatbid = json["seatbid"].as_array().unwrap_or(&empty_array); @@ -225,10 +246,18 @@ impl AdServerMockProvider { let bids = seat["bid"].as_array().unwrap_or(&empty_bids); for bid in bids { - // Mediation layer returns decoded prices for all bids + let slot_id = bid["impid"].as_str().unwrap_or("").to_string(); + + // Recover bidder name from crid ("{bidder}-creative") to look up the + // original SSP bid and restore nurl/burl/ad_id the mediator drops. + let crid = bid["crid"].as_str().unwrap_or(""); + let bidder = crid.strip_suffix("-creative").unwrap_or(""); + let key = (seat_name.to_string(), slot_id.clone(), bidder.to_string()); + let original = bid_index.get(&key); + all_bids.push(Bid { - slot_id: bid["impid"].as_str().unwrap_or("").to_string(), - price: bid["price"].as_f64(), // Now properly decoded by mediation + slot_id, + price: bid["price"].as_f64(), currency: "USD".to_string(), creative: bid["adm"].as_str().map(String::from), width: bid["w"].as_u64().unwrap_or(0) as u32, @@ -239,9 +268,9 @@ impl AdServerMockProvider { .filter_map(|v| v.as_str().map(String::from)) .collect() }), - nurl: None, - burl: None, - ad_id: None, + nurl: original.and_then(|b| b.nurl.clone()), + burl: original.and_then(|b| b.burl.clone()), + ad_id: original.and_then(|b| b.ad_id.clone()), metadata: HashMap::new(), }); } @@ -274,6 +303,19 @@ impl AuctionProvider for AdServerMockProvider { bidder_responses.len() ); + // Build bid index so parse_response can restore nurl/burl/ad_id from + // the original SSP bids (the mock mediator does not echo these fields). + let mut index = BidIndex::new(); + for response in bidder_responses { + for bid in &response.bids { + index.insert( + (response.provider.clone(), bid.slot_id.clone(), bid.bidder.clone()), + bid.clone(), + ); + } + } + *self.bid_index.lock().expect("should lock bid index") = Some(index); + // Build mediation request let mediation_req = self .build_mediation_request(request, bidder_responses) @@ -349,7 +391,15 @@ impl AuctionProvider for AdServerMockProvider { log::trace!("AdServer Mock response: {:?}", response_json); - let auction_response = self.parse_mediation_response(&response_json, response_time_ms); + let bid_index = self + .bid_index + .lock() + .expect("should lock bid index") + .take() + .unwrap_or_default(); + + let auction_response = + self.parse_mediation_response(&response_json, response_time_ms, &bid_index); log::info!( "AdServer Mock returned {} bids in {}ms", @@ -571,7 +621,8 @@ mod tests { "cur": "USD" }); - let auction_response = provider.parse_mediation_response(&mediation_response, 200); + let auction_response = + provider.parse_mediation_response(&mediation_response, 200, &BidIndex::new()); assert_eq!(auction_response.provider, "adserver_mock"); assert_eq!(auction_response.status, BidStatus::Success); @@ -597,7 +648,8 @@ mod tests { "cur": "USD" }); - let auction_response = provider.parse_mediation_response(&mediation_response, 100); + let auction_response = + provider.parse_mediation_response(&mediation_response, 100, &BidIndex::new()); assert_eq!(auction_response.status, BidStatus::NoBid); assert_eq!(auction_response.bids.len(), 0); @@ -791,7 +843,8 @@ mod tests { "cur": "USD" }); - let auction_response = provider.parse_mediation_response(&mediation_response, 200); + let auction_response = + provider.parse_mediation_response(&mediation_response, 200, &BidIndex::new()); assert_eq!(auction_response.status, BidStatus::Success); assert_eq!(auction_response.bids.len(), 2); From 8516caa8e5cd5369755f5edad665311f98fca2dd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 16:17:40 +0530 Subject: [PATCH 057/395] Populate device.user_agent in auction request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit APS reads user agent from request.device — without it, real APS bids arrive with wrong or missing device targeting. Pass the incoming UA from both the page-load and page-bids auction paths. --- crates/trusted-server-core/src/publisher.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 5ab8d1aef..18fc62c8e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -20,7 +20,7 @@ use fastly::{Body, Request, Response}; use crate::auction::orchestrator::{AuctionOrchestrator, DispatchedAuction}; use crate::auction::types::{ - AuctionContext, AuctionRequest, Bid, PublisherInfo, SiteInfo, UserInfo, + AuctionContext, AuctionRequest, Bid, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, }; use crate::backend::BackendConfig; use crate::compat; @@ -935,6 +935,7 @@ pub async fn handle_publisher_request( &request_info, &request_path, co_config, + req.get_header_str("user-agent"), ); let auction_context = AuctionContext { settings, @@ -1180,6 +1181,7 @@ pub(crate) fn build_auction_request( request_info: &crate::http_util::RequestInfo, request_path: &str, co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, + user_agent: Option<&str>, ) -> AuctionRequest { let slots = matched_slots .iter() @@ -1201,7 +1203,11 @@ pub(crate) fn build_auction_request( fresh_id: ec_id.to_string(), consent: Some(consent_context.clone()), }, - device: None, + device: user_agent.filter(|ua| !ua.is_empty()).map(|ua| DeviceInfo { + user_agent: Some(ua.to_string()), + ip: None, + geo: None, + }), site: Some(SiteInfo { domain: request_info.host.clone(), page: page_url, @@ -1456,6 +1462,7 @@ pub async fn handle_page_bids( &request_info, &path_param, co_config, + req.get_header_str("user-agent"), ); let timeout_ms = co_config .auction_timeout_ms From 9e0ec5ba566dd925e01dc967b265142b37e782fb Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 16:31:42 +0530 Subject: [PATCH 058/395] Fix __tsAdInit fallback: look up bids by slot id not div id slotRenderEnded gives a div element id via getSlotElementId(), but __ts_bids is keyed by slot id. Build a divToSlotId map during slot setup (matching the TS implementation) and use it in the event handler. Without this, nurl/burl beacons and hb_adid match checks silently fail in the server-rendered fallback whenever div_id != id. --- crates/trusted-server-core/src/integrations/gpt.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 796d633e1..690ba0486 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -447,22 +447,24 @@ impl IntegrationHeadInjector for GptIntegration { "window.__tsAdInit=function(){", "var slots=window.__ts_ad_slots||[];", "var bids=window.__ts_bids||{};", + "var divToSlotId={};", "googletag.cmd.push(function(){", - "var gptSlots=slots.map(function(slot){", + "slots.map(function(slot){", "var s=googletag.defineSlot(slot.gam_unit_path,slot.formats,slot.div_id);", - "if(!s)return null;", + "if(!s)return;", "s.addService(googletag.pubads());", "Object.entries(slot.targeting||{}).forEach(function(e){s.setTargeting(e[0],e[1]);});", "var b=bids[slot.id]||{};", "[\"hb_pb\",\"hb_bidder\",\"hb_adid\"].forEach(function(k){if(b[k])s.setTargeting(k,b[k]);});", "s.setTargeting(\"ts_initial\",\"1\");", - "return{id:slot.id,gptSlot:s};", - "}).filter(Boolean);", + "divToSlotId[slot.div_id]=slot.id;", + "});", "googletag.pubads().enableSingleRequest();", "googletag.enableServices();", "googletag.pubads().addEventListener(\"slotRenderEnded\",function(ev){", - "var id=ev.slot.getSlotElementId();", - "var b=bids[id]||{};", + "var divId=ev.slot.getSlotElementId();", + "var slotId=divToSlotId[divId]||divId;", + "var b=bids[slotId]||{};", "var ourBidWon=!ev.isEmpty&&b.hb_adid&&ev.slot.getTargeting(\"hb_adid\")[0]===b.hb_adid;", "if(ourBidWon){", "if(b.nurl)navigator.sendBeacon(b.nurl);", From d27a329919e48ea35afc66ad91115f898a3fd10c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 16:32:12 +0530 Subject: [PATCH 059/395] Format lint using cargo fmt --- .../trusted-server-core/src/integrations/adserver_mock.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index c8d0ca7b5..a8f7cadfa 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -309,7 +309,11 @@ impl AuctionProvider for AdServerMockProvider { for response in bidder_responses { for bid in &response.bids { index.insert( - (response.provider.clone(), bid.slot_id.clone(), bid.bidder.clone()), + ( + response.provider.clone(), + bid.slot_id.clone(), + bid.bidder.clone(), + ), bid.clone(), ); } From 790c1232f0efa050eff1ecc5c84a7cd9307a78ea Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 16:44:22 +0530 Subject: [PATCH 060/395] Fix clippy doc-markdown lint in adserver_mock --- crates/trusted-server-core/src/integrations/adserver_mock.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index a8f7cadfa..4330ea660 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -98,7 +98,7 @@ type BidIndex = HashMap<(String, String, String), Bid>; /// Mock ad server mediator provider. pub struct AdServerMockProvider { config: AdServerMockConfig, - /// Bridges SSP bid metadata (nurl/burl/ad_id) from request_bids to parse_response. + /// Bridges SSP bid metadata (`nurl`/`burl`/`ad_id`) from `request_bids` to `parse_response`. bid_index: Mutex>, } From 299f6ba95704dcf0b35a56f2f28d74a7075c6a55 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 15:59:24 +0530 Subject: [PATCH 061/395] Remove inline PBS bidder params from creative-opportunities.toml PBS bidder credentials (mocktioneer, criteo placeholder params) were being sent directly to PBS on every auction request. Per the design spec, PBS bidder params belong in PBS stored requests keyed by slot ID, not in the edge config file. Removes PbsSlotParams struct, SlotProviders.pbs field, the to_ad_slot wiring block, and the corresponding test. Slots without inline bidder params trigger the existing storedrequest fallback path in the Prebid provider. Closes #697 --- .../src/creative_opportunities.rs | 65 ------------------- creative-opportunities.toml | 12 ---- 2 files changed, 77 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 12957d4b8..25add829a 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -125,11 +125,6 @@ impl CreativeOpportunitySlot { serde_json::json!({ "slotID": aps.slot_id }), ); } - if let Some(ref pbs) = self.providers.pbs { - for (bidder_name, params) in &pbs.bidders { - bidders.insert(bidder_name.clone(), params.clone()); - } - } AdSlot { id: self.id.clone(), formats: self @@ -175,8 +170,6 @@ impl CreativeOpportunityFormat { pub struct SlotProviders { /// Amazon Publisher Services (APS/TAM) slot parameters. pub aps: Option, - /// Prebid Server (PBS) slot parameters. - pub pbs: Option, } /// APS-specific parameters for a slot. @@ -186,24 +179,6 @@ pub struct ApsSlotParams { pub slot_id: String, } -/// PBS-specific parameters for a slot. -/// -/// Bidder params are sent inline to Prebid Server so bidder credentials -/// stay in `creative-opportunities.toml` rather than in PBS stored requests. -#[derive(Debug, Clone, Default, Deserialize)] -pub struct PbsSlotParams { - /// Per-bidder params keyed by bidder name (must match PBS adapter name). - /// - /// Example in TOML: - /// ```toml - /// [slot.providers.pbs.bidders] - /// mocktioneer = { bid = 2.00 } - /// criteo = { networkId = 123456, pubid = "123456" } - /// ``` - #[serde(default)] - pub bidders: HashMap, -} - /// TOML file structure for creative opportunity slot definitions. #[derive(Debug, Clone, Deserialize, Default)] pub struct CreativeOpportunitiesFile { @@ -333,46 +308,6 @@ mod tests { ); } - #[test] - fn to_ad_slot_wires_pbs_bidder_params_into_bidders() { - let mut slot = make_slot("atf_sidebar_ad", vec!["/"]); - slot.providers.pbs = Some(PbsSlotParams { - bidders: [ - ( - "mocktioneer".to_string(), - serde_json::json!({ "bid": 2.00 }), - ), - ( - "criteo".to_string(), - serde_json::json!({ "networkId": 123456, "pubid": "123456" }), - ), - ] - .into_iter() - .collect(), - }); - let ad_slot = slot.to_ad_slot("88059007"); - let mock_params = ad_slot - .bidders - .get("mocktioneer") - .expect("should have mocktioneer bidder"); - assert_eq!( - mock_params.get("bid").and_then(serde_json::Value::as_f64), - Some(2.0), - "should wire mocktioneer bid param" - ); - let criteo_params = ad_slot - .bidders - .get("criteo") - .expect("should have criteo bidder"); - assert_eq!( - criteo_params - .get("networkId") - .and_then(serde_json::Value::as_i64), - Some(123456), - "should wire criteo networkId param" - ); - } - #[test] fn to_ad_slot_sets_floor_price_and_formats() { let slot = make_slot("atf", vec!["/"]); diff --git a/creative-opportunities.toml b/creative-opportunities.toml index 95ea849a5..b6ed8900f 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -16,10 +16,6 @@ zone = "atfSidebar" [slot.providers.aps] slot_id = "aps-slot-atf-sidebar" -[slot.providers.pbs.bidders] -mocktioneer = { bid = 2.00 } -criteo = { networkId = 123456, pubid = "123456" } - [[slot]] id = "homepage_header_ad" gam_unit_path = "/88059007/autoblog/homepage" @@ -35,10 +31,6 @@ zone = "header" [slot.providers.aps] slot_id = "aps-slot-homepage-header" -[slot.providers.pbs.bidders] -mocktioneer = { bid = 2.00 } -criteo = { networkId = 123456, pubid = "123456" } - [[slot]] id = "homepage_footer_ad" gam_unit_path = "/88059007/autoblog/homepage" @@ -53,7 +45,3 @@ zone = "fixedBottom" [slot.providers.aps] slot_id = "aps-slot-homepage-footer" - -[slot.providers.pbs.bidders] -mocktioneer = { bid = 1.50 } -criteo = { networkId = 123456, pubid = "123456" } From 03d39f29bc2d643c25b9d85ccbae9fc304ee5d5c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 16:16:51 +0530 Subject: [PATCH 062/395] Clarify and test APS floor price enforcement in mediation path - Rewrite misleading comment in apply_floor_prices: price=None bids pass through in the parallel-only path because decoding is deferred; in the mediation path the mediator decodes prices before this function runs - Add test: decoded APS bid below slot floor is dropped - Add test: decoded APS bid at or above slot floor is kept Closes #698 --- .../src/auction/orchestrator.rs | 83 ++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 17fe405dd..58f46b149 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -542,7 +542,11 @@ impl AuctionOrchestrator { let starting_count = winning_bids.len(); winning_bids.retain(|slot_id, bid| match floor_prices.get(slot_id) { Some(floor) => { - // Bids without price (e.g., APS) pass through - floor checked in mediation + // price=None means the SSP returned an encoded price (e.g. APS amznbid). + // In the parallel-only path this bid cannot yet be floor-checked; it passes + // through and will be decoded (and re-checked) by the mediation layer. + // In the mediation path, mediation decodes prices before calling this + // function, so any bid still carrying price=None is dropped upstream. match bid.price { Some(price) if price >= *floor => true, Some(_) => { @@ -554,7 +558,7 @@ impl AuctionOrchestrator { } None => { log::debug!( - "Passing bid with encoded price for slot '{}' - floor check deferred to mediation", + "Passing encoded-price bid for slot '{}' - price not yet decoded", slot_id ); true @@ -1305,4 +1309,79 @@ mod tests { "Price should still be None (not decoded yet)" ); } + + #[test] + fn test_apply_floor_prices_drops_decoded_aps_bid_below_floor() { + // After mediation decodes an APS bid, apply_floor_prices must enforce the + // slot floor on the resulting price=Some(x) value. This test simulates the + // state of a bid after mediator decoding: price is Some, amznbid is gone. + let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let mut floor_prices = HashMap::new(); + floor_prices.insert("atf".to_string(), 0.50); + + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf".to_string(), + Bid { + slot_id: "atf".to_string(), + price: Some(0.30), // decoded APS price — below $0.50 floor + currency: "USD".to_string(), + creative: Some("
APS Ad
".to_string()), + adomain: None, + bidder: "aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + metadata: HashMap::new(), + }, + ); + + let filtered = orchestrator.apply_floor_prices(winning_bids, &floor_prices); + + assert!( + filtered.is_empty(), + "Decoded APS bid below slot floor should be dropped" + ); + } + + #[test] + fn test_apply_floor_prices_keeps_decoded_aps_bid_at_or_above_floor() { + let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let mut floor_prices = HashMap::new(); + floor_prices.insert("atf".to_string(), 0.50); + + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf".to_string(), + Bid { + slot_id: "atf".to_string(), + price: Some(0.75), // decoded APS price — above floor + currency: "USD".to_string(), + creative: Some("
APS Ad
".to_string()), + adomain: None, + bidder: "aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + metadata: HashMap::new(), + }, + ); + + let filtered = orchestrator.apply_floor_prices(winning_bids, &floor_prices); + + assert_eq!( + filtered.len(), + 1, + "Decoded APS bid at or above floor should be kept" + ); + assert_eq!( + filtered.get("atf").expect("atf should be present").price, + Some(0.75), + "Price should be preserved" + ); + } } From f09eb34ffac5b3230a7a1c4af5804890e9b4954b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 16:24:28 +0530 Subject: [PATCH 063/395] Document and test /auction API contract for non-Prebid.js callers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Expand handle_auction doc: inline-params vs stored-request paths, config passthrough and allowed_context_keys, response headers - Document AdRequest, AdUnit, BidConfig with the stored-request contract: absent/empty bids → empty bidders map → PBS stored-request fallback - Add tests for convert_tsjs_to_auction_request: - No bids → empty bidders map (stored-request path) - Inline bids → bidders map populated - Allowed config key passes through; disallowed key dropped - Invalid 3-element banner size returns error Closes #699 --- .../src/auction/endpoints.rs | 41 +++- .../src/auction/formats.rs | 195 +++++++++++++++++- 2 files changed, 230 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 0430f08ba..5a9ac6f10 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -16,11 +16,44 @@ use super::formats::{convert_to_openrtb_response, convert_tsjs_to_auction_reques use super::types::AuctionContext; use super::AuctionOrchestrator; -/// Handle auction request from /auction endpoint. +/// Handle auction request from `POST /auction`. /// -/// This is the main entry point for running header bidding auctions. -/// It orchestrates bids from multiple providers (Prebid, APS, GAM, etc.) and returns -/// the winning bids in `OpenRTB` format with creative HTML inline in the `adm` field. +/// Accepts a JSON body matching [`AdRequest`][`super::formats::AdRequest`]. +/// The minimum valid request is: +/// +/// ```json +/// { +/// "adUnits": [{ +/// "code": "atf_sidebar_ad", +/// "mediaTypes": { "banner": { "sizes": [[300, 250]] } } +/// }] +/// } +/// ``` +/// +/// ## Bidder params: inline vs. stored-request +/// +/// Each ad unit's `bids` array is **optional**. When absent or empty the PBS +/// integration falls back to a stored-request keyed by the unit's `code` +/// field (`imp.ext.prebid.storedrequest = { id: "" }`). A PBS stored +/// request must therefore exist for every slot code that omits inline params. +/// +/// When `bids` is supplied, each entry's `bidder`/`params` pair is forwarded +/// directly as `imp.ext.prebid.bidder.`. +/// +/// ## Context passthrough (`config`) +/// +/// The optional `config` object is filtered through +/// [`auction.allowed_context_keys`][`crate::settings::AuctionConfig::allowed_context_keys`]. +/// Only keys listed there reach the auction providers (e.g. `"permutive_segments"`). +/// All other keys are silently dropped. Values must be either strings or arrays of +/// strings. +/// +/// ## Response +/// +/// Returns an `OpenRTB 2.x` response. Creative HTML is inlined in each bid's +/// `adm` field after sanitisation and first-party URL rewriting. Response +/// headers include `X-TS-EC` (the caller's Edge Cookie ID) and +/// `X-TS-EC-Fresh` (a freshly generated ID for cookie renewal). /// /// # Errors /// diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index 5237921a7..53c6474a0 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -28,7 +28,11 @@ use super::types::{ PublisherInfo, SiteInfo, UserInfo, }; -/// Request body format for auction endpoints (tsjs/Prebid.js format). +/// Request body for `POST /auction` (tsjs / Prebid.js wire format). +/// +/// `adUnits` lists the placements to bid on. `config` carries optional +/// context values (e.g. audience segments) filtered through +/// [`auction.allowed_context_keys`][`crate::settings::AuctionConfig::allowed_context_keys`]. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AdRequest { @@ -36,6 +40,15 @@ pub struct AdRequest { pub config: Option, } +/// A single ad placement in an [`AdRequest`]. +/// +/// `code` identifies the slot (e.g. `"atf_sidebar_ad"`) and becomes the +/// impression ID in the outgoing `OpenRTB` request. +/// +/// `bids` is optional. When absent or empty the PBS provider falls back to +/// a stored-request keyed by `code` (`imp.ext.prebid.storedrequest.id`). +/// When present, each entry's params are forwarded inline to PBS as +/// `imp.ext.prebid.bidder.`. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AdUnit { @@ -44,7 +57,11 @@ pub struct AdUnit { pub bids: Option>, } -/// Bidder configuration from the request. +/// Inline bidder params for one SSP within an [`AdUnit`]. +/// +/// `params` is passed verbatim to the corresponding PBS bidder adapter. +/// When the `bids` array is absent, the slot falls back to PBS stored +/// requests — see [`AdUnit`] for details. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BidConfig { @@ -318,3 +335,177 @@ pub fn convert_to_openrtb_response( .with_header(HEADER_X_TS_EC_FRESH, &auction_request.user.fresh_id) .with_body(body_bytes)) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::consent::ConsentContext; + use crate::platform::test_support::noop_services; + use crate::test_support::tests::crate_test_settings_str; + use fastly::http::Method; + use fastly::Request; + + fn make_settings() -> Settings { + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings") + } + + fn make_req() -> Request { + Request::new(Method::POST, "https://test-publisher.com/auction") + } + + fn call_convert(body: &AdRequest) -> AuctionRequest { + let settings = make_settings(); + let services = noop_services(); + let req = make_req(); + convert_tsjs_to_auction_request( + body, + &settings, + &services, + &req, + ConsentContext::default(), + "test-ec-id", + None, + ) + .expect("should convert without error") + } + + #[test] + fn no_bids_produces_empty_bidders_map() { + // An ad unit with no `bids` array must produce an empty bidders map. + // An empty bidders map triggers the PBS stored-request fallback: + // the PBS provider sets imp.ext.prebid.storedrequest = { id: "" }. + let body = AdRequest { + ad_units: vec![AdUnit { + code: "atf_sidebar_ad".to_string(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![300, 250]], + }), + }), + bids: None, + }], + config: None, + }; + + let auction_request = call_convert(&body); + + assert_eq!(auction_request.slots.len(), 1, "should have one slot"); + let slot = &auction_request.slots[0]; + assert_eq!(slot.id, "atf_sidebar_ad", "slot id should match unit code"); + assert!( + slot.bidders.is_empty(), + "absent bids array should yield empty bidders map (PBS stored-request path)" + ); + } + + #[test] + fn inline_bids_populate_bidders_map() { + // When bids are supplied, each bidder+params pair should appear in the + // slot's bidders map so PBS receives inline params. + let body = AdRequest { + ad_units: vec![AdUnit { + code: "homepage_header_ad".to_string(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![970, 90]], + }), + }), + bids: Some(vec![BidConfig { + bidder: "kargo".to_string(), + params: serde_json::json!({ "placementId": "client_123" }), + }]), + }], + config: None, + }; + + let auction_request = call_convert(&body); + + let slot = &auction_request.slots[0]; + assert!( + slot.bidders.contains_key("kargo"), + "kargo bidder should be present in slot bidders map" + ); + assert_eq!( + slot.bidders["kargo"]["placementId"], "client_123", + "bidder params should be forwarded verbatim" + ); + } + + #[test] + fn config_allowed_key_passes_through() { + // Keys in auction.allowed_context_keys must reach the auction context. + // The test settings do not set allowed_context_keys so the default + // (empty) applies — verify a key is NOT present rather than IS. + // To test the allow-list, inject a key via a custom settings string. + let settings_str = format!( + "{}\n[auction]\nallowed_context_keys = [\"permutive_segments\"]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&settings_str).expect("should parse"); + let services = noop_services(); + let req = make_req(); + + let body = AdRequest { + ad_units: vec![], + config: Some(serde_json::json!({ + "permutive_segments": ["seg1", "seg2"], + "disallowed_key": "should be dropped", + })), + }; + + let auction_request = convert_tsjs_to_auction_request( + &body, + &settings, + &services, + &req, + ConsentContext::default(), + "test-ec-id", + None, + ) + .expect("should convert"); + + assert!( + auction_request.context.contains_key("permutive_segments"), + "allowed key should be in auction context" + ); + assert!( + !auction_request.context.contains_key("disallowed_key"), + "unlisted key should be dropped" + ); + } + + #[test] + fn invalid_banner_size_returns_error() { + // Banner sizes must be [width, height] pairs; a 3-element size is invalid. + let body = AdRequest { + ad_units: vec![AdUnit { + code: "bad_slot".to_string(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![300, 250, 99]], // invalid — 3 elements + }), + }), + bids: None, + }], + config: None, + }; + + let settings = make_settings(); + let services = noop_services(); + let req = make_req(); + let result = convert_tsjs_to_auction_request( + &body, + &settings, + &services, + &req, + ConsentContext::default(), + "test-ec-id", + None, + ); + + assert!( + result.is_err(), + "3-element banner size should return an error" + ); + } +} From a03c70a8cbbf6065eacb34c5a1ee1ad53c556025 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 17:07:35 +0530 Subject: [PATCH 064/395] Verify and document graceful degradation when no slots match URL - Add debug log at no-match gate in handle_publisher_request and handle_page_bids so operators can confirm the feature is inactive on non-article URLs without reading source code - Add test: empty slots file (kill-switch) returns slots:[] bids:{} - Add test: URL not matching any slot pattern returns slots:[] bids:{} Closes #700 --- crates/trusted-server-core/src/publisher.rs | 117 ++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 18fc62c8e..a4773a629 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -905,6 +905,13 @@ pub async fn handle_publisher_request( let should_run_auction = is_get && !is_prefetch && !is_bot && !matched_slots.is_empty() && consent_allows_auction; + if matched_slots.is_empty() && settings.creative_opportunities.is_some() { + log::debug!( + "No creative opportunity slots matched path '{}' — skipping auction and injection", + request_path + ); + } + let auction_timeout_ms = settings .creative_opportunities .as_ref() @@ -1454,6 +1461,13 @@ pub async fn handle_page_bids( .as_ref() .is_some_and(|tcf| tcf.has_purpose_consent(1)); + if matched_slots.is_empty() { + log::debug!( + "No creative opportunity slots matched path '{}' — skipping auction", + path_param + ); + } + let winning_bids = if !matched_slots.is_empty() && consent_allows_auction { let auction_request = build_auction_request( &matched_slots, @@ -2673,4 +2687,107 @@ mod tests { ); } } + + mod page_bids_no_match_tests { + use super::super::*; + use crate::auction::AuctionOrchestrator; + use crate::creative_opportunities::{ + CreativeOpportunitiesFile, CreativeOpportunityFormat, CreativeOpportunitySlot, + }; + use crate::platform::test_support::noop_services; + use crate::test_support::tests::crate_test_settings_str; + use fastly::http::Method; + use fastly::Request; + + fn settings_with_co() -> Settings { + let toml = format!( + "{}\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") + } + + fn file_with_article_slot() -> CreativeOpportunitiesFile { + CreativeOpportunitiesFile { + slots: vec![CreativeOpportunitySlot { + id: "atf".to_string(), + gam_unit_path: None, + div_id: None, + page_patterns: vec!["/20**".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: crate::auction::types::MediaType::Banner, + }], + floor_price: Some(0.50), + targeting: Default::default(), + providers: Default::default(), + }], + } + } + + fn make_page_bids_request(path: &str) -> Request { + Request::new( + Method::GET, + format!("https://test-publisher.com/_ts/page-bids?path={path}"), + ) + } + + #[tokio::test] + async fn empty_slots_file_returns_empty_slots_and_bids() { + // Spec §8 kill-switch: creative-opportunities.toml with zero slots disables + // all server-side auction activity and injection. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let slots_file = CreativeOpportunitiesFile { slots: vec![] }; + let req = make_page_bids_request("/2024/01/my-article/"); + + let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + .await + .expect("should return ok response"); + + let body: serde_json::Value = + serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + + assert_eq!( + body["slots"].as_array().expect("slots should be array").len(), + 0, + "empty slots file should produce zero injected slots" + ); + assert_eq!( + body["bids"].as_object().expect("bids should be object").len(), + 0, + "empty slots file should produce zero bids" + ); + } + + #[tokio::test] + async fn url_not_matching_any_pattern_returns_empty_response() { + // Slots exist but request path does not match — no auction, no injection. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let slots_file = file_with_article_slot(); // slot matches /20** only + let req = make_page_bids_request("/about"); // does not match + + let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + .await + .expect("should return ok response"); + + let body: serde_json::Value = + serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + + assert_eq!( + body["slots"].as_array().expect("slots should be array").len(), + 0, + "non-matching URL should produce zero injected slots" + ); + assert_eq!( + body["bids"].as_object().expect("bids should be object").len(), + 0, + "non-matching URL should produce zero bids" + ); + } + } } From 421833399efc17692a869e7355d5f105e6b99944 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 17:10:45 +0530 Subject: [PATCH 065/395] Format publisher.rs with cargo fmt --- crates/trusted-server-core/src/publisher.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index a4773a629..235be8178 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -2751,12 +2751,18 @@ mod tests { serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); assert_eq!( - body["slots"].as_array().expect("slots should be array").len(), + body["slots"] + .as_array() + .expect("slots should be array") + .len(), 0, "empty slots file should produce zero injected slots" ); assert_eq!( - body["bids"].as_object().expect("bids should be object").len(), + body["bids"] + .as_object() + .expect("bids should be object") + .len(), 0, "empty slots file should produce zero bids" ); @@ -2779,12 +2785,18 @@ mod tests { serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); assert_eq!( - body["slots"].as_array().expect("slots should be array").len(), + body["slots"] + .as_array() + .expect("slots should be array") + .len(), 0, "non-matching URL should produce zero injected slots" ); assert_eq!( - body["bids"].as_object().expect("bids should be object").len(), + body["bids"] + .as_object() + .expect("bids should be object") + .len(), 0, "non-matching URL should produce zero bids" ); From 0762999f37b977ffe7d61705cac5bb266ce32140 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 17:13:35 +0530 Subject: [PATCH 066/395] Document scroll/refresh handoff contract between TS and slim-Prebid - Clarify in handle_auction doc that /auction is for initial render and programmatic callers; scroll/refresh/SPA navigation is slim-Prebid's domain in Phase 1 - Note Phase 2 slot-template-aware refresh API as deferred future work - Add head_inserts doc clarifying __tsAdInit handles initial render only; slotRenderEnded fires win beacons but does not trigger refresh auctions Closes #702 --- .../trusted-server-core/src/auction/endpoints.rs | 14 ++++++++++++++ crates/trusted-server-core/src/integrations/gpt.rs | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 5a9ac6f10..5d5bb292c 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -55,6 +55,20 @@ use super::AuctionOrchestrator; /// headers include `X-TS-EC` (the caller's Edge Cookie ID) and /// `X-TS-EC-Fresh` (a freshly generated ID for cookie renewal). /// +/// ## Scroll, refresh, and SPA navigation +/// +/// This endpoint is intended for **initial page render** and **programmatic +/// callers** (e.g. slim-Prebid, native apps, server-to-server integrations). +/// It is **not** the intended path for scroll or GPT refresh events. +/// +/// In Phase 1, slim-Prebid owns scroll and refresh: it runs post-`window.load`, +/// listens for GPT refresh events, and runs client-side auctions independently +/// of this endpoint. SPAs that use pushState routing do not trigger TS page-level +/// auctions — slim-Prebid handles those cases too. +/// +/// A slot-template-aware refresh API (`POST /auction/refresh`) is deferred to a +/// future phase and not designed here. +/// /// # Errors /// /// Returns an error if: diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 690ba0486..85ea800ea 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -437,6 +437,19 @@ impl IntegrationHeadInjector for GptIntegration { GPT_INTEGRATION_ID } + /// Injects the `__tsAdInit` bootstrap script into ``. + /// + /// ## Scroll / refresh handoff contract (Phase 1) + /// + /// `__tsAdInit` handles **initial render only**: it wires server-side bid + /// targeting into GPT slots and fires win beacons (`nurl`/`burl`) via + /// `slotRenderEnded`. It does **not** trigger refresh auctions or handle + /// GPT slot refresh events. + /// + /// Post-`window.load`, slim-Prebid takes over: it listens for GPT refresh + /// events, runs client-side auctions, and sets targeting for subsequent + /// impressions. SPA pushState navigation is also slim-Prebid's domain. + /// The `POST /auction` endpoint is not involved in scroll or refresh flows. fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { vec![ ""# @@ -606,7 +624,7 @@ mod tests { request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), ad_slots_script: None, - ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), } } @@ -1263,7 +1281,7 @@ mod tests { ad_slots_script: Some( r#""#.to_string(), ), - ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), }; let mut processor = create_html_processor(config); let output = processor @@ -1287,7 +1305,7 @@ mod tests { fn injects_ts_bids_before_body_close() { let bids_script = r#""#; - let state = std::sync::Arc::new(std::sync::RwLock::new(Some(bids_script.to_string()))); + let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1316,7 +1334,7 @@ mod tests { fn injects_ts_bids_only_once_with_multiple_body_elements() { let bids_script = r#""#; - let state = std::sync::Arc::new(std::sync::RwLock::new(Some(bids_script.to_string()))); + let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1342,7 +1360,7 @@ mod tests { fn injects_empty_ts_bids_when_slots_matched_but_auction_returned_nothing() { // Slots matched (ad_slots_script is Some) but auction task never wrote a result // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. - let state = std::sync::Arc::new(std::sync::RwLock::new(None)); + let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1367,7 +1385,7 @@ mod tests { // No slots matched this URL — ad_slots_script is None. __ts_bids must be // omitted entirely so the publisher's existing client-side GPT flow is // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). - let state = std::sync::Arc::new(std::sync::RwLock::new(None)); + let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 850798e43..71ef2bbe7 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -456,10 +456,16 @@ impl ApsAuctionProvider { aps_response.contextual.slots.len() ); - let slot_map = self - .slot_id_map - .lock() - .expect("should lock APS slot id map"); + // Take the map by value so it does not linger on the provider + // across requests if the Fastly Compute runtime ever reuses Wasm + // instances. Today each request gets its own instance so this is + // belt-and-suspenders; tomorrow it may not be. + let slot_map = std::mem::take( + &mut *self + .slot_id_map + .lock() + .expect("should lock APS slot id map"), + ); for slot in aps_response.contextual.slots { match self.parse_aps_slot(&slot) { Ok(mut bid) => { diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 85ea800ea..cb0994029 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -455,44 +455,21 @@ impl IntegrationHeadInjector for GptIntegration { "" .to_string(), - concat!( - "" - ).to_string(), + format!("", GPT_BOOTSTRAP_JS), ] } } +/// Inline `window.__tsAdInit` bootstrap injected at `` so the bids +/// script at `` can call it before the TSJS bundle has loaded. +/// +/// The bundle's idempotent implementation in +/// `crates/js/lib/src/integrations/gpt/index.ts` later overwrites this stub. +/// Both implementations guard the one-time-per-page setup with +/// `window.__tsServicesEnabled` so neither double-enables services if the +/// publisher's own init code also calls `googletag.enableServices()`. +const GPT_BOOTSTRAP_JS: &str = include_str!("gpt_bootstrap.js"); + // Default value functions fn default_enabled() -> bool { @@ -1120,6 +1097,32 @@ mod tests { ); } + #[test] + fn head_inserts_bootstrap_guards_enable_services_with_idempotency_flag() { + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let combined = integration.head_inserts(&ctx).join(""); + assert!( + combined.contains("__tsServicesEnabled"), + "should guard enableServices/enableSingleRequest with the __tsServicesEnabled flag" + ); + assert!( + combined.contains("window.__tsAdInit"), + "should install __tsAdInit on window" + ); + assert!( + !combined.contains("googletag.pubads().refresh()"), + "should never call unbounded refresh() — only refresh(newSlots)" + ); + } + #[test] fn head_injector_integration_id() { let integration = GptIntegration::new(test_config()); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js new file mode 100644 index 000000000..a3d28a286 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -0,0 +1,78 @@ +// Edge-injected GPT auction bootstrap. +// +// This is the minimal `window.__tsAdInit` that runs on first page load +// before the TSJS bundle has had a chance to install its richer +// idempotent implementation. The bundle in +// crates/js/lib/src/integrations/gpt/index.ts overwrites `__tsAdInit` +// once it loads. +// +// Contract with the bundle: +// - Both implementations must set `window.__tsServicesEnabled = true` +// after calling `enableSingleRequest()`/`enableServices()` so a +// subsequent call from any source (the bundle's `__tsAdInit`, the +// publisher's own GPT init code) becomes a no-op. +// - `refresh()` is called only for the slots defined in this pass, +// never the global slot list, so we never accidentally refresh +// publisher-managed slots that we don't own. +// +// Only installed if `window.__tsAdInit` isn't already defined — that +// way the bundle (or anything else) can preempt this fallback by +// installing first. +(function () { + if (typeof window === "undefined" || window.__tsAdInit) { + return; + } + window.__tsAdInit = function () { + var slots = window.__ts_ad_slots || []; + var bids = window.__ts_bids || {}; + var divToSlotId = {}; + googletag.cmd.push(function () { + var newSlots = []; + slots.forEach(function (slot) { + var s = googletag.defineSlot( + slot.gam_unit_path, + slot.formats, + slot.div_id, + ); + if (!s) return; + s.addService(googletag.pubads()); + Object.entries(slot.targeting || {}).forEach(function (e) { + s.setTargeting(e[0], e[1]); + }); + var b = bids[slot.id] || {}; + ["hb_pb", "hb_bidder", "hb_adid"].forEach(function (k) { + if (b[k]) s.setTargeting(k, b[k]); + }); + s.setTargeting("ts_initial", "1"); + divToSlotId[slot.div_id] = slot.id; + newSlots.push(s); + }); + // Guard the one-time-per-page setup so a follow-up call (e.g. + // publisher's own init code or the bundle's `__tsAdInit` after + // it overwrites this stub) doesn't double-enable services. + if (!window.__tsServicesEnabled) { + googletag.pubads().enableSingleRequest(); + googletag.enableServices(); + window.__tsServicesEnabled = true; + googletag + .pubads() + .addEventListener("slotRenderEnded", function (ev) { + var divId = ev.slot.getSlotElementId(); + var slotId = divToSlotId[divId] || divId; + var b = (window.__ts_bids || {})[slotId] || {}; + var ourBidWon = + !ev.isEmpty && + b.hb_adid && + ev.slot.getTargeting("hb_adid")[0] === b.hb_adid; + if (ourBidWon) { + if (b.nurl) navigator.sendBeacon(b.nurl); + if (b.burl) navigator.sendBeacon(b.burl); + } + }); + } + if (newSlots.length > 0) { + googletag.pubads().refresh(newSlots); + } + }); + }; +})(); diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index d7711d61e..b74b234ca 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -164,10 +164,6 @@ pub struct PrebidIntegrationConfig { /// - `both` — consent in both cookies and body (default) #[serde(default)] pub consent_forwarding: ConsentForwardingMode, - /// When true, suppresses client-side nurl firing. - /// Use for PBS deployments that fire nurl internally. - #[serde(default)] - pub suppress_nurl: bool, } impl IntegrationConfig for PrebidIntegrationConfig { @@ -1661,16 +1657,9 @@ mod tests { bid_param_overrides: HashMap::default(), bid_param_override_rules: Vec::new(), consent_forwarding: ConsentForwardingMode::Both, - suppress_nurl: false, } } - #[test] - fn prebid_config_suppress_nurl_defaults_to_false() { - let config = base_config(); - assert!(!config.suppress_nurl, "should not suppress nurl by default"); - } - fn create_test_auction_request() -> AuctionRequest { AuctionRequest { id: "auction-123".to_string(), diff --git a/crates/trusted-server-core/src/price_bucket.rs b/crates/trusted-server-core/src/price_bucket.rs index b683020bf..cfdca9eb4 100644 --- a/crates/trusted-server-core/src/price_bucket.rs +++ b/crates/trusted-server-core/src/price_bucket.rs @@ -20,7 +20,10 @@ impl PriceGranularity { #[must_use] pub fn price_bucket(cpm: f64, granularity: PriceGranularity) -> String { - if cpm <= 0.0 { + // Reject NaN / Inf early so the `(x * 100.0).floor() as u64` cast below + // can never see a non-finite value (the cast's behaviour for NaN/Inf is + // implementation-defined in Rust and "saturate to 0" only by convention). + if !cpm.is_finite() || cpm <= 0.0 { return "0.00".to_string(); } match granularity { @@ -125,4 +128,31 @@ mod tests { price_bucket(2.53, PriceGranularity::Dense) ); } + + #[test] + fn non_finite_cpm_returns_zero_bucket() { + for granularity in [ + PriceGranularity::Dense, + PriceGranularity::Low, + PriceGranularity::Medium, + PriceGranularity::High, + PriceGranularity::Auto, + ] { + assert_eq!( + price_bucket(f64::NAN, granularity), + "0.00", + "NaN cpm should bucket to 0.00 for granularity {granularity:?}" + ); + assert_eq!( + price_bucket(f64::INFINITY, granularity), + "0.00", + "+Inf cpm should bucket to 0.00 for granularity {granularity:?}" + ); + assert_eq!( + price_bucket(f64::NEG_INFINITY, granularity), + "0.00", + "-Inf cpm should bucket to 0.00 for granularity {granularity:?}" + ); + } + } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 494f06eb5..8908eaf09 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -12,7 +12,7 @@ //! content-rewriting concern. use std::io::Write; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex}; use error_stack::{Report, ResultExt}; use fastly::http::{header, StatusCode}; @@ -39,6 +39,11 @@ use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, S use crate::streaming_replacer::create_url_replacer; const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; +/// Read buffer size for streaming body processing and brotli internal buffers. +/// Both the `Decompressor` and `CompressorWriter` use this value so all +/// brotli I/O layers operate on consistently-sized chunks. +const STREAM_CHUNK_SIZE: usize = 8192; + fn restrict_accept_encoding(req: &mut Request) { // If the client sent no Accept-Encoding, leave the request unchanged so the // origin responds without compression. Adding encodings here would cause the @@ -194,7 +199,7 @@ struct ProcessResponseParams<'a> { content_type: &'a str, integration_registry: &'a IntegrationRegistry, ad_slots_script: Option<&'a str>, - ad_bids_state: &'a Arc>>, + ad_bids_state: &'a Arc>>, } /// Process response body through the streaming pipeline. @@ -262,26 +267,32 @@ fn process_response_streaming( Ok(()) } -/// Create a unified HTML stream processor +/// Create a unified HTML stream processor. +/// +/// Builds the config via [`HtmlProcessorConfig::from_settings`] and then +/// layers the auction-hold streaming fields on top via +/// [`HtmlProcessorConfig::with_ad_state`], so the canonical builder stays the +/// single source of truth: a future field added to `from_settings` is +/// inherited here automatically. fn create_html_stream_processor( origin_host: &str, request_host: &str, request_scheme: &str, - _settings: &Settings, + settings: &Settings, integration_registry: &IntegrationRegistry, ad_slots_script: Option, - ad_bids_state: Arc>>, + ad_bids_state: Arc>>, ) -> Result> { use crate::html_processor::{create_html_processor, HtmlProcessorConfig}; - let config = HtmlProcessorConfig { - origin_host: origin_host.to_string(), - request_host: request_host.to_string(), - request_scheme: request_scheme.to_string(), - integrations: integration_registry.clone(), - ad_slots_script, - ad_bids_state, - }; + let config = HtmlProcessorConfig::from_settings( + settings, + integration_registry, + origin_host, + request_host, + request_scheme, + ) + .with_ad_state(ad_slots_script, ad_bids_state); Ok(create_html_processor(config)) } @@ -412,7 +423,7 @@ pub struct OwnedProcessResponseParams { pub(crate) request_scheme: String, pub(crate) content_type: String, pub(crate) ad_slots_script: Option, - pub(crate) ad_bids_state: Arc>>, + pub(crate) ad_bids_state: Arc>>, /// In-flight SSP bids dispatched before `pending_origin.wait()`. /// The streaming phase collects these and writes bids to `ad_bids_state` /// before processing the last body chunk, so `` injection sees live bids. @@ -493,7 +504,7 @@ pub async fn stream_publisher_body_async( if !is_html { // Non-HTML: collect auction first, then stream. There is no // to hold, so delaying the entire body until collection is acceptable. - let placeholder = Request::get("https://placeholder.invalid/"); + let placeholder = Request::get(crate::auction::types::MEDIATOR_PLACEHOLDER_URL); let result = orchestrator .collect_dispatched_auction( dispatched, @@ -540,16 +551,25 @@ pub async fn stream_publisher_body_async( .await } -/// Build a minimal [`AuctionContext`] for the mediator call in collection. +/// Build a minimal [`AuctionContext`] for the collect phase. /// -/// The `request` field is a short-lived placeholder (providers use it only for -/// header extraction; the placeholder is functionally equivalent to the original -/// since `req` was already consumed by `send_async` before dispatch). +/// See [`AuctionContext::request`]: the orchestrator's collect path runs +/// after `send_async` has already consumed the real client request, so this +/// context carries a synthetic placeholder. The orchestrator itself +/// instantiates a fresh placeholder when it actually invokes a mediator — +/// this argument is plumbing for the (presently unused) case where the +/// orchestrator needs the caller's request shape. fn make_collect_context<'a>( settings: &'a Settings, services: &'a RuntimeServices, placeholder: &'a Request, ) -> AuctionContext<'a> { + debug_assert_eq!( + placeholder.get_url_str(), + crate::auction::types::MEDIATOR_PLACEHOLDER_URL, + "make_collect_context must be given the canonical placeholder; \ + callers must not forward a real client request through the collect path" + ); AuctionContext { settings, request: placeholder, @@ -560,27 +580,87 @@ fn make_collect_context<'a>( } } +/// Well-known crawler User-Agent fragments. Best-effort: an attacker can +/// trivially spoof their UA, so this is for opt-out signalling to honest +/// crawlers (preventing SSP auctions burning partner quota on their behalf), +/// not security. +pub(crate) const BOT_USER_AGENT_FRAGMENTS: &[&str] = + &["Googlebot", "Bingbot", "AhrefsBot", "SemrushBot", "DotBot"]; + +/// Returns true when the request's User-Agent matches any well-known crawler +/// fragment in [`BOT_USER_AGENT_FRAGMENTS`]. +pub(crate) fn is_bot_user_agent(req: &Request) -> bool { + let ua = req.get_header_str("user-agent").unwrap_or(""); + BOT_USER_AGENT_FRAGMENTS + .iter() + .any(|frag| ua.contains(frag)) +} + +/// Returns true when the request advertises itself as a prefetch via either +/// the standard `Sec-Purpose` or the legacy `Purpose` header. +pub(crate) fn is_prefetch_request(req: &Request) -> bool { + req.get_header_str("sec-purpose") + .is_some_and(|v| v.contains("prefetch")) + || req + .get_header_str("purpose") + .is_some_and(|v| v.contains("prefetch")) +} + /// Write winning bids from an auction result into the shared `ad_bids_state` lock. pub(crate) fn write_bids_to_state( winning_bids: &std::collections::HashMap, price_granularity: PriceGranularity, - ad_bids_state: &Arc>>, + ad_bids_state: &Arc>>, ) { - log::info!( + log::debug!( "write_bids_to_state: {} winning bid(s): [{}]", winning_bids.len(), winning_bids.keys().cloned().collect::>().join(", ") ); let bid_map = build_bid_map(winning_bids, price_granularity); let bids_script = build_bids_script(&bid_map); - *ad_bids_state.write().expect("should write bid state") = Some(bids_script); + *ad_bids_state.lock().expect("should lock bid state") = Some(bids_script); +} + +/// Prepend an HTML comment summarising the auction result onto the shared +/// `ad_bids_state` so it lands directly before the injected bids `` sequences inside the string. pub(crate) fn build_bids_script(bid_map: &serde_json::Map) -> String { - let json = serde_json::to_string(bid_map).unwrap_or_else(|_| "{}".to_string()); + let json = serde_json::to_string(bid_map) + .expect("serde_json::to_string of Map should be infallible"); let escaped = html_escape_for_script(&json); format!( "", @@ -1338,7 +1385,8 @@ pub(crate) fn build_ad_slots_script( }) }) .collect(); - let json = serde_json::to_string(&slots).unwrap_or_else(|_| "[]".to_string()); + let json = serde_json::to_string(&slots) + .expect("serde_json::to_string of Vec should be infallible"); let escaped = html_escape_for_script(&json); format!( "", @@ -1473,58 +1521,74 @@ pub async fn handle_page_bids( .as_ref() .is_some_and(|tcf| tcf.has_purpose_consent(1)); + // Same bot / prefetch guards the publisher path uses — without them this + // endpoint would fire real SSP auctions on Sec-Purpose=prefetch warm-up + // navigations and known crawler UA scans, burning partner request quota. + let is_prefetch = is_prefetch_request(&req); + let is_bot = is_bot_user_agent(&req); + if matched_slots.is_empty() { log::debug!( "No creative opportunity slots matched path '{}' — skipping auction", path_param ); + } else if is_bot || is_prefetch { + log::debug!( + "page-bids: skipping auction for path '{}' (is_bot={}, is_prefetch={})", + path_param, + is_bot, + is_prefetch + ); } - let winning_bids = if !matched_slots.is_empty() && consent_allows_auction { - let mut auction_request = build_auction_request( - &matched_slots, - &ec_id, - &consent_context, - &request_info, - &path_param, - co_config, - req.get_header_str("user-agent"), - ); - auction_request.user.eids = parse_ts_eids_cookie(cookie_jar.as_ref()); - let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); - if client_ip.is_some() || geo.is_some() { - let device = auction_request.device.get_or_insert(DeviceInfo { - user_agent: None, - ip: None, - geo: None, - }); - device.ip = client_ip; - device.geo = geo.clone(); - } - let timeout_ms = co_config - .auction_timeout_ms - .unwrap_or(settings.auction.timeout_ms); - let auction_context = AuctionContext { - settings, - request: &req, - client_info: services.client_info(), - timeout_ms, - provider_responses: None, - services, - }; - match orchestrator - .run_auction(&auction_request, &auction_context, services) - .await - { - Ok(result) => result.winning_bids, - Err(e) => { - log::warn!("page-bids auction failed: {e:?}"); - std::collections::HashMap::new() + let winning_bids = + if !matched_slots.is_empty() && consent_allows_auction && !is_bot && !is_prefetch { + let slots_ctx = MatchedSlotsContext { + matched_slots: &matched_slots, + request_path: &path_param, + }; + let mut auction_request = build_auction_request( + &slots_ctx, + &ec_id, + &consent_context, + &request_info, + req.get_header_str("user-agent"), + ); + auction_request.user.eids = parse_ts_eids_cookie(cookie_jar.as_ref()); + let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); + if client_ip.is_some() || geo.is_some() { + let device = auction_request.device.get_or_insert(DeviceInfo { + user_agent: None, + ip: None, + geo: None, + }); + device.ip = client_ip; + device.geo = geo.clone(); } - } - } else { - std::collections::HashMap::new() - }; + let timeout_ms = co_config + .auction_timeout_ms + .unwrap_or(settings.auction.timeout_ms); + let auction_context = AuctionContext { + settings, + request: &req, + client_info: services.client_info(), + timeout_ms, + provider_responses: None, + services, + }; + match orchestrator + .run_auction(&auction_request, &auction_context, services) + .await + { + Ok(result) => result.winning_bids, + Err(e) => { + log::warn!("page-bids auction failed: {e:?}"); + std::collections::HashMap::new() + } + } + } else { + std::collections::HashMap::new() + }; let bid_map = build_bid_map(&winning_bids, co_config.price_granularity); @@ -2223,7 +2287,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(RwLock::new(None)), + ad_bids_state: Arc::new(Mutex::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), }; @@ -2268,7 +2332,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(RwLock::new(None)), + ad_bids_state: Arc::new(Mutex::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), }; @@ -2304,7 +2368,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(RwLock::new(None)), + ad_bids_state: Arc::new(Mutex::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), }; @@ -2407,7 +2471,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(RwLock::new(None)), + ad_bids_state: Arc::new(Mutex::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), }; @@ -2461,7 +2525,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(RwLock::new(None)), + ad_bids_state: Arc::new(Mutex::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), }; @@ -2527,6 +2591,7 @@ mod tests { .into_iter() .collect(), providers: Default::default(), + compiled_patterns: Vec::new(), } } @@ -2745,6 +2810,7 @@ mod tests { floor_price: Some(0.50), targeting: Default::default(), providers: Default::default(), + compiled_patterns: Vec::new(), }], } } @@ -2791,6 +2857,79 @@ mod tests { ); } + #[tokio::test] + async fn bot_user_agent_returns_slots_but_no_bids() { + // Crawlers should get slot definitions (so HTML structure is unchanged) + // but the server must not burn SSP request quota running a real auction + // for them. Same gate the publisher path applies. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let slots_file = file_with_article_slot(); + let mut req = make_page_bids_request("/2024/01/my-article/"); + req.set_header("user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1)"); + + let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + .await + .expect("should return ok response"); + + let body: serde_json::Value = + serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 1, + "bot request should still get slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "bot request must not run an auction (no SSP cost burned for crawlers)" + ); + } + + #[tokio::test] + async fn prefetch_request_returns_slots_but_no_bids() { + // Navigations triggered by Sec-Purpose=prefetch should not fire real + // SSP auctions — the user has not yet visited the page. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let slots_file = file_with_article_slot(); + let mut req = make_page_bids_request("/2024/01/my-article/"); + req.set_header("sec-purpose", "prefetch"); + + let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + .await + .expect("should return ok response"); + + let body: serde_json::Value = + serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 1, + "prefetch request should still get slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "prefetch request must not run an auction" + ); + } + #[tokio::test] async fn url_not_matching_any_pattern_returns_empty_response() { // Slots exist but request path does not match — no auction, no injection. From 93c1678d2ce13be9a3cad7e30954ed7a4cba0394 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 23 May 2026 23:02:54 +0530 Subject: [PATCH 070/395] Address PR review findings from #680 - Fix gpt_bootstrap.js APS beacon miss: listener now uses hb_bidder fallback when hb_adid is absent, matching the bundle's slotRenderEnded logic - Fix stale divToSlotId in inline listener: read from window.__tsDivToSlotId dynamically instead of local closure so SPA navigation updates are seen; early-return for slots not managed by Trusted Server - Populate window.__tsPrevGptSlots and window.__tsDivToSlotId from inline bootstrap so bundle's destroySlots and SPA nav path have correct state - Call installSlimPrebidLoader() in module init so the slim-Prebid lazy loader activates when __tsjs_slim_prebid_url is set; add three Vitest cases - Update /auction doc comment to distinguish /__ts/page-bids (SPA navigation) from /auction (initial render) and slim-Prebid (scroll/refresh) --- crates/js/lib/src/integrations/gpt/index.ts | 1 + .../lib/test/integrations/gpt/index.test.ts | 51 +++++++++++++++++++ .../src/auction/endpoints.rs | 12 +++-- .../src/integrations/gpt_bootstrap.js | 17 +++++-- 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index fee79c1b6..611b0aeac 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -396,4 +396,5 @@ if (typeof window !== 'undefined') { installTsAdInit(); installSpaAuctionHook(); + installSlimPrebidLoader(); } diff --git a/crates/js/lib/test/integrations/gpt/index.test.ts b/crates/js/lib/test/integrations/gpt/index.test.ts index 57c4015dc..839b121d6 100644 --- a/crates/js/lib/test/integrations/gpt/index.test.ts +++ b/crates/js/lib/test/integrations/gpt/index.test.ts @@ -165,6 +165,57 @@ describe('GPT shim – patchCommandQueue', () => { }); }); +describe('GPT – installSlimPrebidLoader', () => { + type SlimWindow = Window & { __tsjs_slim_prebid_url?: string }; + + afterEach(() => { + delete (window as SlimWindow).__tsjs_slim_prebid_url; + }); + + it('is a no-op when __tsjs_slim_prebid_url is not set', async () => { + const { installSlimPrebidLoader } = await import('../../../src/integrations/gpt/index'); + const addEventListenerSpy = vi.spyOn(window, 'addEventListener'); + installSlimPrebidLoader(); + expect(addEventListenerSpy).not.toHaveBeenCalledWith('load', expect.any(Function)); + addEventListenerSpy.mockRestore(); + }); + + it('appends a deferred script tag when __tsjs_slim_prebid_url is set and load fires', async () => { + (window as SlimWindow).__tsjs_slim_prebid_url = 'https://cdn.example.com/slim-prebid.js'; + const { installSlimPrebidLoader } = await import('../../../src/integrations/gpt/index'); + + installSlimPrebidLoader(); + + // Simulate the window load event. + window.dispatchEvent(new Event('load')); + + const scripts = Array.from(document.querySelectorAll('script[defer]')); + const injected = scripts.find( + (s) => (s as HTMLScriptElement).src === 'https://cdn.example.com/slim-prebid.js' + ); + expect(injected).toBeDefined(); + + // Clean up + injected?.parentNode?.removeChild(injected); + }); + + it('module init calls installSlimPrebidLoader — script injected when URL is preset', async () => { + vi.resetModules(); + (window as SlimWindow).__tsjs_slim_prebid_url = 'https://cdn.example.com/slim-prebid-init.js'; + + await import('../../../src/integrations/gpt/index'); + window.dispatchEvent(new Event('load')); + + const scripts = Array.from(document.querySelectorAll('script[defer]')); + const injected = scripts.find( + (s) => (s as HTMLScriptElement).src === 'https://cdn.example.com/slim-prebid-init.js' + ); + expect(injected).toBeDefined(); + + injected?.parentNode?.removeChild(injected); + }); +}); + describe('GPT shim – runtime gating', () => { type GatedWindow = Window & { __tsjs_gpt_enabled?: boolean; diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 5b4e7b259..22fc11e8e 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -61,10 +61,14 @@ use super::AuctionOrchestrator; /// callers** (e.g. slim-Prebid, native apps, server-to-server integrations). /// It is **not** the intended path for scroll or GPT refresh events. /// -/// In Phase 1, slim-Prebid owns scroll and refresh: it runs post-`window.load`, -/// listens for GPT refresh events, and runs client-side auctions independently -/// of this endpoint. SPAs that use pushState routing do not trigger TS page-level -/// auctions — slim-Prebid handles those cases too. +/// **SPA navigation** is handled by `GET /__ts/page-bids`: the client-side SPA +/// hook (`installSpaAuctionHook`) intercepts `pushState`/`replaceState`/`popstate` +/// events and calls that endpoint to fetch fresh slots and bids for each new +/// route, then invokes `window.__tsAdInit()` with the updated data. +/// +/// **Scroll and GPT refresh** are owned by slim-Prebid in Phase 1: it runs +/// post-`window.load`, listens for GPT refresh events, and runs client-side +/// auctions independently of this endpoint. /// /// A slot-template-aware refresh API (`POST /auction/refresh`) is deferred to a /// future phase and not designed here. diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index a3d28a286..85109d724 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -47,6 +47,11 @@ divToSlotId[slot.div_id] = slot.id; newSlots.push(s); }); + // Expose slot metadata on window so later calls (SPA navigation, + // the bundle's __tsAdInit) can destroy stale slots and the render + // listener can resolve slot IDs after navigation updates these maps. + window.__tsPrevGptSlots = newSlots; + window.__tsDivToSlotId = divToSlotId; // Guard the one-time-per-page setup so a follow-up call (e.g. // publisher's own init code or the bundle's `__tsAdInit` after // it overwrites this stub) doesn't double-enable services. @@ -58,12 +63,18 @@ .pubads() .addEventListener("slotRenderEnded", function (ev) { var divId = ev.slot.getSlotElementId(); - var slotId = divToSlotId[divId] || divId; + // Read from window so SPA navigation updates are picked up; + // early-return for slots not managed by Trusted Server. + var slotId = (window.__tsDivToSlotId || {})[divId]; + if (!slotId) return; var b = (window.__ts_bids || {})[slotId] || {}; + // Prebid: verify the specific creative via hb_adid targeting. + // APS: no hb_adid — fire if any TS bidder is present and slot is non-empty. var ourBidWon = !ev.isEmpty && - b.hb_adid && - ev.slot.getTargeting("hb_adid")[0] === b.hb_adid; + (b.hb_adid + ? ev.slot.getTargeting("hb_adid")[0] === b.hb_adid + : !!b.hb_bidder); if (ourBidWon) { if (b.nurl) navigator.sendBeacon(b.nurl); if (b.burl) navigator.sendBeacon(b.burl); From 0346330905b3e5e8487ecb566baac6c2504d5214 Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Tue, 26 May 2026 09:42:33 -0700 Subject: [PATCH 071/395] Formatting --- crates/trusted-server-core/src/integrations/sourcepoint.rs | 4 ++-- creative-opportunities.toml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/sourcepoint.rs b/crates/trusted-server-core/src/integrations/sourcepoint.rs index a911b5d5a..adea7b446 100644 --- a/crates/trusted-server-core/src/integrations/sourcepoint.rs +++ b/crates/trusted-server-core/src/integrations/sourcepoint.rs @@ -1073,9 +1073,9 @@ mod tests { let integration = SourcepointIntegration::new(Arc::new(config(true))); let document_state = IntegrationDocumentState::default(); let ctx = IntegrationHtmlContext { - request_host: "ts.autoblog.com", + request_host: "ts.examnple.com", request_scheme: "https", - origin_host: "origin.autoblog.com", + origin_host: "origin.examnple.com", document_state: &document_state, }; diff --git a/creative-opportunities.toml b/creative-opportunities.toml index b6ed8900f..da1ed23e7 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -3,7 +3,7 @@ [[slot]] id = "atf_sidebar_ad" -gam_unit_path = "/88059007/autoblog/news" +gam_unit_path = "/a/b/news" div_id = "ad-atf_sidebar-0-_r_2_" page_patterns = ["/20**", "/news/**"] formats = [{ width = 300, height = 250 }] @@ -18,7 +18,7 @@ slot_id = "aps-slot-atf-sidebar" [[slot]] id = "homepage_header_ad" -gam_unit_path = "/88059007/autoblog/homepage" +gam_unit_path = "/a/b/homepage" div_id = "ad-header-0-_R_jpalubtak5lb_" page_patterns = ["/"] formats = [{ width = 970, height = 90 }, { width = 728, height = 90 }, { width = 970, height = 250 }] @@ -33,7 +33,7 @@ slot_id = "aps-slot-homepage-header" [[slot]] id = "homepage_footer_ad" -gam_unit_path = "/88059007/autoblog/homepage" +gam_unit_path = "/a/b/homepage" div_id = "ad-fixed_bottom-0-_R_klubtak5lb_" page_patterns = ["/"] formats = [{ width = 728, height = 90 }] From ca98985b003eb70931e5f52d8ff734deac92b140 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 28 May 2026 15:36:29 +0530 Subject: [PATCH 072/395] Address pass-4 review findings (#680) - Fix examnple.com typo in sourcepoint.rs test fixture - Guard SPA navigation race: check inflight controller identity after await res.json() before writing __ts_ad_slots/__ts_bids - Extract build_empty_bids_script() helper; html_processor.rs now calls it instead of duplicating the inline literal - Add invariant comment to unreachable None branch of prepend_auction_debug_comment - Cap parse_ts_eids_cookie to 32 eids / 32 uids per eid; log and return None when exceeded - Add #[serde(deny_unknown_fields)] to openrtb::Eid and Uid - Add #[serde(deny_unknown_fields)] to CreativeOpportunitiesFile and CreativeOpportunitySlot - Log debug message when adserver_mock crid does not match -creative convention - Skip zero-dimension bids in adserver_mock with debug log - Fail closed in APS parse_aps_slot on malformed size string instead of producing 0x0 bid --- crates/js/lib/src/integrations/gpt/index.ts | 1 + crates/trusted-server-core/src/cookies.rs | 8 +++++++- .../src/creative_opportunities.rs | 2 ++ .../trusted-server-core/src/html_processor.rs | 11 +++++----- .../src/integrations/adserver_mock.rs | 20 ++++++++++++++++--- .../src/integrations/aps.rs | 12 ++++++++++- .../src/integrations/sourcepoint.rs | 4 ++-- crates/trusted-server-core/src/openrtb.rs | 2 ++ crates/trusted-server-core/src/publisher.rs | 10 ++++++++++ 9 files changed, 57 insertions(+), 13 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 611b0aeac..e1a1ee267 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -328,6 +328,7 @@ export function installSpaAuctionHook(): void { }); if (!res.ok) return; const data = (await res.json()) as PageBidsResponse; + if (inflight !== controller) return; win.__ts_ad_slots = data.slots; win.__ts_bids = data.bids; win.__tsAdInit?.(); diff --git a/crates/trusted-server-core/src/cookies.rs b/crates/trusted-server-core/src/cookies.rs index 4f0e7f9c0..91f92d830 100644 --- a/crates/trusted-server-core/src/cookies.rs +++ b/crates/trusted-server-core/src/cookies.rs @@ -141,7 +141,13 @@ pub(crate) fn parse_ts_eids_cookie(jar: Option<&CookieJar>) -> Option>(&decoded) { - Ok(eids) if !eids.is_empty() => Some(eids), + Ok(eids) if !eids.is_empty() => { + if eids.len() > 32 || eids.iter().any(|e| e.uids.len() > 32) { + log::debug!("ts-eids cookie: too many eids or uids, rejecting"); + return None; + } + Some(eids) + } Ok(_) => None, Err(e) => { log::debug!("ts-eids cookie: JSON parse failed: {e}"); diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index cbf79b114..95180041e 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -42,6 +42,7 @@ pub struct CreativeOpportunitiesConfig { /// A single ad placement opportunity on the publisher's site. #[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] pub struct CreativeOpportunitySlot { /// Unique identifier for the slot (e.g., `"atf"`, `"below-fold-sidebar"`). pub id: String, @@ -224,6 +225,7 @@ pub struct ApsSlotParams { /// TOML file structure for creative opportunity slot definitions. #[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] pub struct CreativeOpportunitiesFile { /// All slot definitions in the file (mapped from `[[slot]]` TOML arrays). #[serde(rename = "slot", default)] diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index c54c46897..6005e3cc3 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -34,6 +34,7 @@ use crate::integrations::{ IntegrationHtmlContext, IntegrationHtmlPostProcessor, IntegrationRegistry, IntegrationScriptContext, ScriptRewriteAction, }; +use crate::publisher::build_empty_bids_script; use crate::settings::Settings; use crate::streaming_processor::{HtmlRewriterAdapter, StreamProcessor}; use crate::tsjs; @@ -328,21 +329,19 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let state = state.clone(); let injected_bids = injected_bids.clone(); if let Some(handlers) = el.end_tag_handlers() { - let handler: EndTagHandler<'static> = Box::new( - move |end_tag: &mut EndTag<'_>| { + let handler: EndTagHandler<'static> = + Box::new(move |end_tag: &mut EndTag<'_>| { if injected_bids.swap(true, Ordering::SeqCst) { return Ok(()); } let script_guard = state.lock().expect("should lock bid state"); let bids_script = match &*script_guard { Some(s) => s.clone(), - None => r#""# - .to_string(), + None => build_empty_bids_script(), }; end_tag.before(&bids_script, ContentType::Html); Ok(()) - }, - ); + }); handlers.push(handler); } Ok(()) diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 1d968484c..beacef1df 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -251,17 +251,31 @@ impl AdServerMockProvider { // Recover bidder name from crid ("{bidder}-creative") to look up the // original SSP bid and restore nurl/burl/ad_id the mediator drops. let crid = bid["crid"].as_str().unwrap_or(""); - let bidder = crid.strip_suffix("-creative").unwrap_or(""); + let bidder = crid.strip_suffix("-creative").unwrap_or_else(|| { + log::debug!( + "adserver_mock: crid '{crid}' does not match '-creative' — dropping nurl/burl/ad_id" + ); + "" + }); let key = (seat_name.to_string(), slot_id.clone(), bidder.to_string()); let original = bid_index.get(&key); + let width = bid["w"].as_u64().unwrap_or(0) as u32; + let height = bid["h"].as_u64().unwrap_or(0) as u32; + if width == 0 || height == 0 { + log::debug!( + "adserver_mock: bid for slot '{slot_id}' has zero dimension ({width}×{height}), skipping" + ); + continue; + } + all_bids.push(Bid { slot_id, price: bid["price"].as_f64(), currency: "USD".to_string(), creative: bid["adm"].as_str().map(String::from), - width: bid["w"].as_u64().unwrap_or(0) as u32, - height: bid["h"].as_u64().unwrap_or(0) as u32, + width, + height, bidder: seat_name.to_string(), adomain: bid["adomain"].as_array().map(|arr| { arr.iter() diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 71ef2bbe7..304f61a06 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -406,7 +406,17 @@ impl ApsAuctionProvider { } // Parse size from "WxH" format - let (width, height) = Self::parse_size(&slot.size).unwrap_or((0, 0)); + let (width, height) = match Self::parse_size(&slot.size) { + Some(dims) => dims, + None => { + log::debug!( + "APS: slot '{}' has malformed size '{}', skipping", + slot.slot_id, + slot.size + ); + return Err(()); + } + }; // Build metadata from targeting keys - includes encoded price for mediation let mut metadata = HashMap::new(); diff --git a/crates/trusted-server-core/src/integrations/sourcepoint.rs b/crates/trusted-server-core/src/integrations/sourcepoint.rs index adea7b446..b48075a6a 100644 --- a/crates/trusted-server-core/src/integrations/sourcepoint.rs +++ b/crates/trusted-server-core/src/integrations/sourcepoint.rs @@ -1073,9 +1073,9 @@ mod tests { let integration = SourcepointIntegration::new(Arc::new(config(true))); let document_state = IntegrationDocumentState::default(); let ctx = IntegrationHtmlContext { - request_host: "ts.examnple.com", + request_host: "ts.example.com", request_scheme: "https", - origin_host: "origin.examnple.com", + origin_host: "origin.example.com", document_state: &document_state, }; diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 63d63435c..aff580608 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -76,6 +76,7 @@ pub struct ConsentedProvidersSettings { /// An Extended User ID entry from an identity provider. #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Eid { /// Identity provider domain (e.g. `"id5-sync.com"`). pub source: String, @@ -85,6 +86,7 @@ pub struct Eid { /// A single user identifier within an [`Eid`] entry. #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Uid { /// The identifier value. pub id: String, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8908eaf09..c6ffa7761 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -651,6 +651,8 @@ pub(crate) fn prepend_auction_debug_comment( *script = format!("{debug_comment}\n{script}"); } None => { + // invariant: write_bids_to_state is always called before this and + // always sets Some(_); this branch is unreachable in production. *state = Some(debug_comment); } } @@ -1353,6 +1355,14 @@ pub(crate) fn build_bids_script(bid_map: &serde_json::Map` tag used when no bids were returned. +/// +/// Shares the same shape as [`build_bids_script`] so any change to the script +/// format stays in one place. +pub(crate) fn build_empty_bids_script() -> String { + build_bids_script(&serde_json::Map::new()) +} + /// Build the `__ts_ad_slots` ``. + /// Pre-computed ``. /// Injected at `` open. `None` when no slots matched. pub ad_slots_script: Option, /// Shared auction result — written by auction task before HTML processing begins. /// Handler reads this in `el.on_end_tag()` on the body element. - /// `None` means no auction ran; inject empty `__ts_bids = {}` as fallback. + /// `None` means no auction ran; inject empty `tsjs.bids = {}` as fallback. pub ad_bids_state: std::sync::Arc>>, } @@ -311,10 +311,10 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso Ok(()) } }), - // Inject __ts_bids before via end_tag_handlers — only when + // Inject tsjs.bids before via end_tag_handlers — only when // slots matched this URL. When no slots matched, skip injection entirely // so the publisher's existing client-side Prebid/GPT flow is unmodified - // (dual-mode rollout: calling __tsAdInit with empty slots would invoke + // (dual-mode rollout: calling tsjs.adInit with empty slots would invoke // enableSingleRequest/enableServices and conflict with the publisher's GPT init). // Guard with AtomicBool so the script is only injected once even if // the origin HTML contains multiple elements (e.g. template fragments). @@ -1278,7 +1278,8 @@ mod tests { request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: Some( - r#""#.to_string(), + r#""# + .to_string(), ), ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), }; @@ -1291,8 +1292,12 @@ mod tests { .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert!( - html.contains("window.__ts_ad_slots"), - "should inject ad slots at head-open" + html.contains("window.tsjs=window.tsjs||{}"), + "should inject ad slots namespace at head-open" + ); + assert!( + html.contains(".adSlots=JSON.parse"), + "should inject adSlots at head-open" ); assert!( !html.contains("__ts_request_id"), @@ -1302,15 +1307,16 @@ mod tests { #[test] fn injects_ts_bids_before_body_close() { - let bids_script = - r#""#; + let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: Some("".to_string()), + ad_slots_script: Some( + r#""#.to_string(), + ), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1319,27 +1325,32 @@ mod tests { .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert!( - html.contains("window.__ts_bids"), + html.contains("window.tsjs=window.tsjs||{}"), + "should inject _ts namespace for bids before " + ); + assert!( + html.contains(".bids=JSON.parse"), "should inject bids before " ); let bids_pos = html - .find("window.__ts_bids") - .expect("bids should be in output"); + .find("window.tsjs=window.tsjs||{}") + .expect("bids namespace should be in output"); let body_close_pos = html.find("").expect(" should be in output"); assert!(bids_pos < body_close_pos, "bids must appear before "); } #[test] fn injects_ts_bids_only_once_with_multiple_body_elements() { - let bids_script = - r#""#; + let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: Some("".to_string()), + ad_slots_script: Some( + r#""#.to_string(), + ), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1349,9 +1360,9 @@ mod tests { .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert_eq!( - html.matches("window.__ts_bids").count(), + html.matches(".bids=JSON.parse").count(), 1, - "should inject __ts_bids exactly once even with multiple elements" + "should inject tsjs.bids exactly once even with multiple elements" ); } @@ -1365,7 +1376,9 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: Some("".to_string()), + ad_slots_script: Some( + r#""#.to_string(), + ), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1374,14 +1387,14 @@ mod tests { .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert!( - html.contains("__ts_bids=JSON.parse(\"{}\")"), + html.contains(".bids=JSON.parse(\"{}\")"), "should inject empty bids fallback when auction produced nothing" ); } #[test] fn does_not_inject_ts_bids_when_no_slots_matched() { - // No slots matched this URL — ad_slots_script is None. __ts_bids must be + // No slots matched this URL — ad_slots_script is None. tsjs.bids must be // omitted entirely so the publisher's existing client-side GPT flow is // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). let state = std::sync::Arc::new(std::sync::Mutex::new(None)); @@ -1399,8 +1412,8 @@ mod tests { .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert!( - !html.contains("__ts_bids"), - "should NOT inject __ts_bids when no slots matched" + !html.contains(".bids=JSON.parse"), + "should NOT inject tsjs.bids when no slots matched" ); } } diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index beacef1df..483e4499c 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -89,7 +89,7 @@ impl IntegrationConfig for AdServerMockConfig { // ============================================================================ /// Lookup index built from original SSP bids during `request_bids`, consumed -/// during `parse_response` to restore `nurl`/`burl`/`ad_id` that the mock +/// during `parse_response` to restore render/accounting fields that the mock /// mediator endpoint does not echo back. /// /// Keyed by `(provider_name, slot_id, bidder_name)`. @@ -98,7 +98,7 @@ type BidIndex = HashMap<(String, String, String), Bid>; /// Mock ad server mediator provider. pub struct AdServerMockProvider { config: AdServerMockConfig, - /// Bridges SSP bid metadata (`nurl`/`burl`/`ad_id`) from `request_bids` to `parse_response`. + /// Bridges SSP bid metadata from `request_bids` to `parse_response`. bid_index: Mutex>, } @@ -226,7 +226,7 @@ impl AdServerMockProvider { /// Mediation returns decoded prices for all bids (including APS bids that were encoded). /// /// `bid_index` is the SSP-bid lookup built in `request_bids`. The mock mediator - /// does not echo `nurl`/`burl`/`ad_id` back, so they are restored from the index + /// does not echo render/accounting fields back, so they are restored from the index /// using `(seat, impid, bidder)` where bidder is recovered from the echoed `crid` /// field (`"{bidder}-creative"` format set during request construction). fn parse_mediation_response( @@ -249,16 +249,18 @@ impl AdServerMockProvider { let slot_id = bid["impid"].as_str().unwrap_or("").to_string(); // Recover bidder name from crid ("{bidder}-creative") to look up the - // original SSP bid and restore nurl/burl/ad_id the mediator drops. + // original SSP bid and restore render/accounting fields the mediator drops. let crid = bid["crid"].as_str().unwrap_or(""); let bidder = crid.strip_suffix("-creative").unwrap_or_else(|| { log::debug!( - "adserver_mock: crid '{crid}' does not match '-creative' — dropping nurl/burl/ad_id" + "adserver_mock: crid '{crid}' does not match '-creative'; render/accounting fields may be missing" ); "" }); let key = (seat_name.to_string(), slot_id.clone(), bidder.to_string()); let original = bid_index.get(&key); + let restored_bidder = + original.map_or_else(|| seat_name.to_string(), |b| b.bidder.clone()); let width = bid["w"].as_u64().unwrap_or(0) as u32; let height = bid["h"].as_u64().unwrap_or(0) as u32; @@ -276,7 +278,7 @@ impl AdServerMockProvider { creative: bid["adm"].as_str().map(String::from), width, height, - bidder: seat_name.to_string(), + bidder: restored_bidder, adomain: bid["adomain"].as_array().map(|arr| { arr.iter() .filter_map(|v| v.as_str().map(String::from)) @@ -285,6 +287,9 @@ impl AdServerMockProvider { nurl: original.and_then(|b| b.nurl.clone()), burl: original.and_then(|b| b.burl.clone()), ad_id: original.and_then(|b| b.ad_id.clone()), + cache_id: original.and_then(|b| b.cache_id.clone()), + cache_host: original.and_then(|b| b.cache_host.clone()), + cache_path: original.and_then(|b| b.cache_path.clone()), metadata: HashMap::new(), }); } @@ -563,6 +568,9 @@ mod tests { nurl: None, burl: None, ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: HashMap::new(), }], response_time_ms: 150, @@ -583,6 +591,9 @@ mod tests { nurl: None, burl: None, ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: HashMap::new(), }], response_time_ms: 120, @@ -656,6 +667,98 @@ mod tests { assert_eq!(bid.height, 90); } + #[test] + fn parse_mediation_response_restores_original_bid_render_fields() { + let provider = AdServerMockProvider::new(AdServerMockConfig::default()); + let mediation_response = json!({ + "id": "test-auction-123", + "seatbid": [ + { + "seat": "prebid", + "bid": [ + { + "id": "mediated-bid-001", + "impid": "header-banner", + "price": 0.20, + "adm": "
Mediated Ad
", + "w": 728, + "h": 90, + "crid": "mocktioneer-creative", + "adomain": ["example.com"] + } + ] + } + ], + "cur": "USD" + }); + let mut bid_index = BidIndex::new(); + bid_index.insert( + ( + "prebid".to_string(), + "header-banner".to_string(), + "mocktioneer".to_string(), + ), + Bid { + slot_id: "header-banner".to_string(), + price: Some(0.20), + currency: "USD".to_string(), + creative: Some("
Original Ad
".to_string()), + adomain: Some(vec!["example.com".to_string()]), + bidder: "mocktioneer".to_string(), + width: 728, + height: 90, + nurl: Some("https://ssp.example/win".to_string()), + burl: Some("https://ssp.example/bill".to_string()), + ad_id: Some("bid-impression-id".to_string()), + cache_id: Some("cache-uuid".to_string()), + cache_host: Some("cache.example".to_string()), + cache_path: Some("/cache".to_string()), + metadata: HashMap::new(), + }, + ); + + let auction_response = + provider.parse_mediation_response(&mediation_response, 42, &bid_index); + + assert_eq!(auction_response.status, BidStatus::Success); + assert_eq!(auction_response.bids.len(), 1); + let bid = &auction_response.bids[0]; + assert_eq!( + bid.bidder, "mocktioneer", + "should preserve underlying bidder for hb_bidder targeting" + ); + assert_eq!( + bid.nurl.as_deref(), + Some("https://ssp.example/win"), + "should restore nurl" + ); + assert_eq!( + bid.burl.as_deref(), + Some("https://ssp.example/bill"), + "should restore burl" + ); + assert_eq!( + bid.ad_id.as_deref(), + Some("bid-impression-id"), + "should restore ad_id" + ); + assert_eq!( + bid.cache_id.as_deref(), + Some("cache-uuid"), + "should restore PBS cache UUID" + ); + assert_eq!( + bid.cache_host.as_deref(), + Some("cache.example"), + "should restore PBS cache host" + ); + assert_eq!( + bid.cache_path.as_deref(), + Some("/cache"), + "should restore PBS cache path" + ); + } + #[test] fn test_parse_empty_mediation_response() { let config = AdServerMockConfig::default(); @@ -727,6 +830,9 @@ mod tests { nurl: None, burl: None, ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: aps_metadata, }], response_time_ms: 100, diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 304f61a06..d1c449bf5 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -451,6 +451,9 @@ impl ApsAuctionProvider { nurl: None, // Real APS uses client-side event tracking burl: None, ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata, }) } diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index cb0994029..5f88f69c6 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -81,6 +81,16 @@ pub struct GptConfig { /// Whether to rewrite GPT script URLs in publisher HTML. #[serde(default = "default_rewrite_script")] pub rewrite_script: bool, + + /// URL for the slim-Prebid bundle loaded post-window.load. + /// + /// When set, `installSlimPrebidLoader()` in the GPT bundle will load this + /// script after `window.load`, enabling scroll/refresh client-side auctions + /// and userID module warm-up. Set to the publisher's tsjs-prebid bundle URL. + /// + /// Override via env var: `TRUSTED_SERVER__INTEGRATIONS__GPT__SLIM_PREBID_URL` + #[serde(default, skip_serializing_if = "Option::is_none")] + pub slim_prebid_url: Option, } impl IntegrationConfig for GptConfig { @@ -437,11 +447,11 @@ impl IntegrationHeadInjector for GptIntegration { GPT_INTEGRATION_ID } - /// Injects the `__tsAdInit` bootstrap script into ``. + /// Injects the `tsjs.adInit` bootstrap script into ``. /// /// ## Scroll / refresh handoff contract (Phase 1) /// - /// `__tsAdInit` handles **initial render only**: it wires server-side bid + /// `tsjs.adInit` handles **initial render only**: it wires server-side bid /// targeting into GPT slots and fires win beacons (`nurl`/`burl`) via /// `slotRenderEnded`. It does **not** trigger refresh auctions or handle /// GPT slot refresh events. @@ -451,22 +461,31 @@ impl IntegrationHeadInjector for GptIntegration { /// impressions. SPA pushState navigation is also slim-Prebid's domain. /// The `POST /auction` endpoint is not involved in scroll or refresh flows. fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - vec![ + let mut scripts = vec![ "" .to_string(), format!("", GPT_BOOTSTRAP_JS), - ] + ]; + + if let Some(ref url) = self.config.slim_prebid_url { + scripts.push(format!( + "", + serde_json::to_string(url).expect("should serialize string") + )); + } + + scripts } } -/// Inline `window.__tsAdInit` bootstrap injected at `` so the bids +/// Inline `window.tsjs.adInit` bootstrap injected at `` so the bids /// script at `` can call it before the TSJS bundle has loaded. /// /// The bundle's idempotent implementation in /// `crates/js/lib/src/integrations/gpt/index.ts` later overwrites this stub. /// Both implementations guard the one-time-per-page setup with -/// `window.__tsServicesEnabled` so neither double-enables services if the +/// `window.tsjs.servicesEnabled` so neither double-enables services if the /// publisher's own init code also calls `googletag.enableServices()`. const GPT_BOOTSTRAP_JS: &str = include_str!("gpt_bootstrap.js"); @@ -502,6 +521,7 @@ mod tests { script_url: default_script_url(), cache_ttl_seconds: 3600, rewrite_script: true, + slim_prebid_url: None, } } @@ -1062,10 +1082,10 @@ mod tests { }; let inserts = integration.head_inserts(&ctx); let combined = inserts.join(""); - assert!(combined.contains("__tsAdInit"), "should define __tsAdInit"); + assert!(combined.contains("ts.adInit"), "should define tsjs.adInit"); assert!( - combined.contains("window.__ts_bids"), - "should read window.__ts_bids synchronously" + combined.contains("ts.bids"), + "should read tsjs.bids synchronously" ); assert!( combined.contains("ts_initial"), @@ -1110,13 +1130,10 @@ mod tests { }; let combined = integration.head_inserts(&ctx).join(""); assert!( - combined.contains("__tsServicesEnabled"), - "should guard enableServices/enableSingleRequest with the __tsServicesEnabled flag" - ); - assert!( - combined.contains("window.__tsAdInit"), - "should install __tsAdInit on window" + combined.contains("ts.servicesEnabled"), + "should guard enableServices/enableSingleRequest with the tsjs.servicesEnabled flag" ); + assert!(combined.contains("ts.adInit"), "should install tsjs.adInit"); assert!( !combined.contains("googletag.pubads().refresh()"), "should never call unbounded refresh() — only refresh(newSlots)" @@ -1131,4 +1148,59 @@ mod tests { "gpt" ); } + + #[test] + fn head_inserts_emits_slim_prebid_url_when_configured() { + let config = GptConfig { + slim_prebid_url: Some("https://cdn.example.com/tsjs-prebid.min.js".to_string()), + ..test_config() + }; + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + + let inserts = integration.head_inserts(&ctx); + + assert_eq!( + inserts.len(), + 3, + "should emit three head inserts when slim_prebid_url is set" + ); + assert_eq!( + inserts[2], + r#""#, + "should emit the slim-Prebid URL as a JSON-encoded string assignment" + ); + } + + #[test] + fn head_inserts_omits_slim_prebid_url_when_not_configured() { + let integration = GptIntegration::new(test_config()); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + + let inserts = integration.head_inserts(&ctx); + + assert_eq!( + inserts.len(), + 2, + "should emit exactly two head inserts when slim_prebid_url is absent" + ); + assert!( + inserts + .iter() + .all(|s| !s.contains("__tsjs_slim_prebid_url")), + "should not emit slim-Prebid URL tag when not configured" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 85109d724..0c7ea0dd2 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -1,88 +1,108 @@ // Edge-injected GPT auction bootstrap. // -// This is the minimal `window.__tsAdInit` that runs on first page load +// This is the minimal `window.tsjs.adInit` that runs on first page load // before the TSJS bundle has had a chance to install its richer // idempotent implementation. The bundle in -// crates/js/lib/src/integrations/gpt/index.ts overwrites `__tsAdInit` +// crates/js/lib/src/integrations/gpt/index.ts overwrites `tsjs.adInit` // once it loads. // // Contract with the bundle: -// - Both implementations must set `window.__tsServicesEnabled = true` +// - Both implementations must set `window.tsjs.servicesEnabled = true` // after calling `enableSingleRequest()`/`enableServices()` so a -// subsequent call from any source (the bundle's `__tsAdInit`, the -// publisher's own GPT init code) becomes a no-op. +// subsequent call becomes a no-op. // - `refresh()` is called only for the slots defined in this pass, -// never the global slot list, so we never accidentally refresh -// publisher-managed slots that we don't own. +// never the global slot list. // -// Only installed if `window.__tsAdInit` isn't already defined — that -// way the bundle (or anything else) can preempt this fallback by -// installing first. +// Only installed if `window.tsjs.adInit` isn't already defined. (function () { - if (typeof window === "undefined" || window.__tsAdInit) { - return; - } - window.__tsAdInit = function () { - var slots = window.__ts_ad_slots || []; - var bids = window.__ts_bids || {}; + if (typeof window === "undefined") return; + var ts = (window.tsjs = window.tsjs || {}); + if (ts.adInit) return; + + ts.adInit = function () { + var slots = ts.adSlots || []; + var bids = ts.bids || {}; var divToSlotId = {}; + googletag.cmd.push(function () { + // Slots TS defined itself — tracked for SPA destroy. Publisher-owned + // slots are reused but never destroyed by TS on navigation. var newSlots = []; + // All slots to refresh (TS-defined + publisher-owned reused). + var slotsToRefresh = []; slots.forEach(function (slot) { - var s = googletag.defineSlot( - slot.gam_unit_path, - slot.formats, - slot.div_id, - ); - if (!s) return; - s.addService(googletag.pubads()); + // Resolve actual div ID: exact match first, then prefix query. + // div_id in config may be a stable prefix (e.g. "ad-header-0-") when + // the suffix is dynamically generated by the framework at render time. + var el = + document.getElementById(slot.div_id) || + document.querySelector( + "[id^='" + slot.div_id + "']:not([id$='-container'])", + ); + if (!el) return; + var actualDivId = el.id; + var b = bids[slot.id] || {}; + + var existingSlots = googletag.pubads().getSlots(); + var s = + existingSlots.find(function (gs) { + return gs.getSlotElementId() === actualDivId; + }) || null; + var tsOwned = false; + if (!s) { + // Use outer container div for TS's slot when publisher hasn't defined + // theirs yet — keeps both slots on separate divs so publisher's + // later defineSlot on the inner div doesn't conflict. + var containerEl = document.getElementById(actualDivId + "-container"); + var slotDivId = containerEl ? containerEl.id : actualDivId; + s = googletag.defineSlot(slot.gam_unit_path, slot.formats, slotDivId); + if (!s) return; + s.addService(googletag.pubads()); + tsOwned = true; + } + Object.entries(slot.targeting || {}).forEach(function (e) { s.setTargeting(e[0], e[1]); }); - var b = bids[slot.id] || {}; - ["hb_pb", "hb_bidder", "hb_adid"].forEach(function (k) { + [ + "hb_pb", + "hb_bidder", + "hb_adid", + "hb_cache_host", + "hb_cache_path", + ].forEach(function (k) { if (b[k]) s.setTargeting(k, b[k]); }); + // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts s.setTargeting("ts_initial", "1"); - divToSlotId[slot.div_id] = slot.id; - newSlots.push(s); + divToSlotId[actualDivId] = slot.id; + if (tsOwned) newSlots.push(s); + slotsToRefresh.push(s); }); - // Expose slot metadata on window so later calls (SPA navigation, - // the bundle's __tsAdInit) can destroy stale slots and the render - // listener can resolve slot IDs after navigation updates these maps. - window.__tsPrevGptSlots = newSlots; - window.__tsDivToSlotId = divToSlotId; - // Guard the one-time-per-page setup so a follow-up call (e.g. - // publisher's own init code or the bundle's `__tsAdInit` after - // it overwrites this stub) doesn't double-enable services. - if (!window.__tsServicesEnabled) { + ts.prevGptSlots = newSlots; + ts.divToSlotId = divToSlotId; + if (!ts.servicesEnabled) { googletag.pubads().enableSingleRequest(); googletag.enableServices(); - window.__tsServicesEnabled = true; - googletag - .pubads() - .addEventListener("slotRenderEnded", function (ev) { - var divId = ev.slot.getSlotElementId(); - // Read from window so SPA navigation updates are picked up; - // early-return for slots not managed by Trusted Server. - var slotId = (window.__tsDivToSlotId || {})[divId]; - if (!slotId) return; - var b = (window.__ts_bids || {})[slotId] || {}; - // Prebid: verify the specific creative via hb_adid targeting. - // APS: no hb_adid — fire if any TS bidder is present and slot is non-empty. - var ourBidWon = - !ev.isEmpty && - (b.hb_adid - ? ev.slot.getTargeting("hb_adid")[0] === b.hb_adid - : !!b.hb_bidder); - if (ourBidWon) { - if (b.nurl) navigator.sendBeacon(b.nurl); - if (b.burl) navigator.sendBeacon(b.burl); - } - }); + ts.servicesEnabled = true; + googletag.pubads().addEventListener("slotRenderEnded", function (ev) { + var divId = ev.slot.getSlotElementId(); + var slotId = (ts.divToSlotId || {})[divId]; + if (!slotId) return; + var b = (ts.bids || {})[slotId] || {}; + var ourBidWon = + !ev.isEmpty && + (b.hb_adid + ? ev.slot.getTargeting("hb_adid")[0] === b.hb_adid + : !!b.hb_bidder); + if (ourBidWon) { + if (b.nurl) navigator.sendBeacon(b.nurl); + if (b.burl) navigator.sendBeacon(b.burl); + } + }); } - if (newSlots.length > 0) { - googletag.pubads().refresh(newSlots); + if (slotsToRefresh.length > 0) { + googletag.pubads().refresh(slotsToRefresh); } }); }; diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index b74b234ca..1ab937baa 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -8,6 +8,7 @@ use fastly::http::{header, Method, StatusCode, Url}; use fastly::{Request, Response}; use serde::{Deserialize, Serialize}; use serde_json::Value as Json; +use url::Url as ParsedUrl; use validator::Validate; use crate::auction::provider::AuctionProvider; @@ -1374,6 +1375,49 @@ impl PrebidAuctionProvider { .collect() }); + // Extract PBS Cache coordinates from ext.prebid.cache.bids + let cache_entry = bid_obj + .get("ext") + .and_then(|e| e.get("prebid")) + .and_then(|p| p.get("cache")) + .and_then(|c| c.get("bids")); + + let cache_id = cache_entry + .and_then(|c| c.get("cacheId")) + .and_then(|v| v.as_str()) + .map(String::from); + + let (cache_host, cache_path) = cache_entry + .and_then(|c| c.get("url")) + .and_then(|v| v.as_str()) + .and_then(|url_str| { + ParsedUrl::parse(url_str) + .map_err(|e| log::debug!("PBS cache URL parse failed: {}", e)) + .ok() + }) + .map(|u| { + let host = u.host_str().map(String::from); + // path() returns "/" for root — only use if non-trivial + let path = u.path().to_string(); + let path = if path.is_empty() || path == "/" { + None + } else { + Some(path) + }; + (host, path) + }) + .unwrap_or((None, None)); + + // Guard: if we extracted a cache UUID but couldn't extract the host, + // the bid will have hb_adid set but no endpoint to fetch from — creative will fail. + if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — \ + creative will fail to render for slot '{}'", + slot_id + ); + } + Ok(AuctionBid { slot_id, price: Some(price), // Prebid provides decoded prices @@ -1386,6 +1430,9 @@ impl PrebidAuctionProvider { nurl, burl, ad_id, + cache_id, + cache_host, + cache_path, metadata: std::collections::HashMap::new(), }) } @@ -4339,4 +4386,137 @@ set = { networkId = 42 } "should fail fast when a canonical rule has no matcher fields" ); } + + #[test] + fn parse_bid_extracts_cache_id_from_ext_prebid_cache_bids() { + let bid_json = serde_json::json!({ + "id": "bid-id-123", + "impid": "atf_sidebar_ad", + "price": 1.50, + "adm": "
ad
", + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://openads.adsrvr.org/cache?uuid=f47447a0-b759-4f2f-9887-af458b79b570", + "cacheId": "f47447a0-b759-4f2f-9887-af458b79b570" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "thetradedesk") + .expect("should parse bid"); + assert_eq!( + bid.cache_id.as_deref(), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should extract cacheId as cache_id" + ); + assert_eq!( + bid.cache_host.as_deref(), + Some("openads.adsrvr.org"), + "should extract host from cache URL" + ); + assert_eq!( + bid.cache_path.as_deref(), + Some("/cache"), + "should extract path from cache URL" + ); + } + + #[test] + fn parse_bid_sets_cache_fields_to_none_when_no_cache_entry() { + let bid_json = serde_json::json!({ + "id": "bid-id-456", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250 + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert!(bid.cache_id.is_none(), "should be None when cache absent"); + assert!(bid.cache_host.is_none(), "should be None when cache absent"); + assert!(bid.cache_path.is_none(), "should be None when cache absent"); + } + + #[test] + fn parse_bid_handles_malformed_cache_url_gracefully() { + let bid_json = serde_json::json!({ + "id": "bid-id-789", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "not-a-valid-url", + "cacheId": "some-uuid" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid without panicking"); + assert_eq!( + bid.cache_id.as_deref(), + Some("some-uuid"), + "should still extract cacheId even if URL is malformed" + ); + assert!( + bid.cache_host.is_none(), + "should be None when URL parse fails" + ); + assert!( + bid.cache_path.is_none(), + "should be None when URL parse fails" + ); + } + + #[test] + fn parse_bid_preserves_ad_id_alongside_cache_id() { + let bid_json = serde_json::json!({ + "id": "bid-impression-id", + "impid": "atf_sidebar_ad", + "adid": "bidder-ad-id-abc", + "price": 1.0, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://cache.example.com/cache", + "cacheId": "cache-uuid-xyz" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert_eq!( + bid.ad_id.as_deref(), + Some("bidder-ad-id-abc"), + "should keep ad_id from adid field" + ); + assert_eq!( + bid.cache_id.as_deref(), + Some("cache-uuid-xyz"), + "should extract cache UUID separately" + ); + } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index c6ffa7761..12a368c47 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -428,7 +428,7 @@ pub struct OwnedProcessResponseParams { /// The streaming phase collects these and writes bids to `ad_bids_state` /// before processing the last body chunk, so `` injection sees live bids. pub(crate) dispatched_auction: Option, - /// Price granularity used to bucket bids when building `__ts_bids`. + /// Price granularity used to bucket bids when building `tsjs.bids`. pub(crate) price_granularity: PriceGranularity, } @@ -516,6 +516,7 @@ pub async fn stream_publisher_body_async( &result.winning_bids, params.price_granularity, ¶ms.ad_bids_state, + settings.debug.inject_adm_for_testing, ); return stream_publisher_body(body, output, params, settings, integration_registry); } @@ -611,13 +612,14 @@ pub(crate) fn write_bids_to_state( winning_bids: &std::collections::HashMap, price_granularity: PriceGranularity, ad_bids_state: &Arc>>, + inject_adm: bool, ) { log::debug!( "write_bids_to_state: {} winning bid(s): [{}]", winning_bids.len(), winning_bids.keys().cloned().collect::>().join(", ") ); - let bid_map = build_bid_map(winning_bids, price_granularity); + let bid_map = build_bid_map(winning_bids, price_granularity, inject_adm); let bids_script = build_bids_script(&bid_map); *ad_bids_state.lock().expect("should lock bid state") = Some(bids_script); } @@ -757,7 +759,12 @@ async fn one_behind_loop( "one_behind_loop: collect complete — {} winning bid(s)", result.winning_bids.len() ); - write_bids_to_state(&result.winning_bids, price_granularity, ad_bids_state); + write_bids_to_state( + &result.winning_bids, + price_granularity, + ad_bids_state, + settings.debug.inject_adm_for_testing, + ); if settings.debug.auction_html_comment { prepend_auction_debug_comment("stream", &result, ad_bids_state); @@ -853,7 +860,7 @@ pub async fn handle_publisher_request( integration_registry: &IntegrationRegistry, services: &RuntimeServices, orchestrator: &AuctionOrchestrator, - slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, + slots: &[crate::creative_opportunities::CreativeOpportunitySlot], mut req: Request, ) -> Result> { log::debug!("Proxying request to publisher_origin"); @@ -939,7 +946,7 @@ pub async fn handle_publisher_request( let is_bot = is_bot_user_agent(&req); let matched_slots: Vec<_> = if settings.creative_opportunities.is_some() && is_get { - crate::creative_opportunities::match_slots(&slots_file.slots, &request_path) + crate::creative_opportunities::match_slots(slots, &request_path) .into_iter() .cloned() .collect() @@ -1192,7 +1199,12 @@ pub async fn handle_publisher_request( "BufferedProcessed: auction collected — {} winning bid(s)", result.winning_bids.len() ); - write_bids_to_state(&result.winning_bids, price_granularity, &ad_bids_state); + write_bids_to_state( + &result.winning_bids, + price_granularity, + &ad_bids_state, + settings.debug.inject_adm_for_testing, + ); if settings.debug.auction_html_comment { prepend_auction_debug_comment("buffered", &result, &ad_bids_state); @@ -1311,6 +1323,7 @@ fn html_escape_for_script(s: &str) -> String { pub(crate) fn build_bid_map( winning_bids: &std::collections::HashMap, granularity: crate::price_bucket::PriceGranularity, + include_adm: bool, ) -> serde_json::Map { winning_bids .iter() @@ -1323,10 +1336,30 @@ pub(crate) fn build_bid_map( "hb_bidder".to_string(), serde_json::Value::String(bid.bidder.clone()), ); - if let Some(ref ad_id) = bid.ad_id { + // hb_adid: use PBS Cache UUID when present — the Prebid Universal Creative uses + // this as the cache lookup key, NOT the OpenRTB bid ID (bid.ad_id). Fall back to + // bid.ad_id for APS and other non-PBS providers. + let hb_adid = bid.cache_id.as_deref().or(bid.ad_id.as_deref()); + if let Some(id) = hb_adid { obj.insert( "hb_adid".to_string(), - serde_json::Value::String(ad_id.clone()), + serde_json::Value::String(id.to_string()), + ); + } + + // Cache endpoint coordinates — only present for PBS bids with Prebid Cache enabled. + // The Prebid Universal Creative constructs: + // https://?uuid= + if let Some(ref host) = bid.cache_host { + obj.insert( + "hb_cache_host".to_string(), + serde_json::Value::String(host.clone()), + ); + } + if let Some(ref path) = bid.cache_path { + obj.insert( + "hb_cache_path".to_string(), + serde_json::Value::String(path.clone()), ); } if let Some(ref nurl) = bid.nurl { @@ -1335,13 +1368,40 @@ pub(crate) fn build_bid_map( if let Some(ref burl) = bid.burl { obj.insert("burl".to_string(), serde_json::Value::String(burl.clone())); } + // Include raw creative markup only for explicit debug injection. + // The pbRender bridge can use it while PBS Cache is unavailable. + if include_adm { + if let Some(ref adm) = bid.creative { + obj.insert("adm".to_string(), serde_json::Value::String(adm.clone())); + } + obj.insert( + "debug_bid".to_string(), + serde_json::json!({ + "slot_id": bid.slot_id, + "price": bid.price, + "currency": bid.currency, + "creative": bid.creative, + "adomain": bid.adomain, + "bidder": bid.bidder, + "width": bid.width, + "height": bid.height, + "nurl": bid.nurl, + "burl": bid.burl, + "ad_id": bid.ad_id, + "cache_id": bid.cache_id, + "cache_host": bid.cache_host, + "cache_path": bid.cache_path, + "metadata": bid.metadata, + }), + ); + } (slot_id.clone(), serde_json::Value::Object(obj)) }) }) .collect() } -/// Build the `__ts_bids` `` sequences inside the string. @@ -1350,7 +1410,7 @@ pub(crate) fn build_bids_script(bid_map: &serde_json::Map should be infallible"); let escaped = html_escape_for_script(&json); format!( - "", + "", escaped ) } @@ -1363,7 +1423,7 @@ pub(crate) fn build_empty_bids_script() -> String { build_bids_script(&serde_json::Map::new()) } -/// Build the `__ts_ad_slots` `", + "", escaped ) } @@ -1479,7 +1539,7 @@ pub async fn handle_page_bids( settings: &Settings, orchestrator: &AuctionOrchestrator, services: &RuntimeServices, - slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, + slots: &[crate::creative_opportunities::CreativeOpportunitySlot], req: Request, ) -> Result> { let Some(co_config) = &settings.creative_opportunities else { @@ -1494,11 +1554,10 @@ pub async fn handle_page_bids( .map(|(_, v)| v.into_owned()) .unwrap_or_else(|| "/".to_string()); - let matched_slots: Vec<_> = - crate::creative_opportunities::match_slots(&slots_file.slots, &path_param) - .into_iter() - .cloned() - .collect(); + let matched_slots: Vec<_> = crate::creative_opportunities::match_slots(slots, &path_param) + .into_iter() + .cloned() + .collect(); let http_req = compat::from_fastly_headers_ref(&req); let request_info = @@ -1600,7 +1659,11 @@ pub async fn handle_page_bids( std::collections::HashMap::new() }; - let bid_map = build_bid_map(&winning_bids, co_config.price_granularity); + let bid_map = build_bid_map( + &winning_bids, + co_config.price_granularity, + settings.debug.inject_adm_for_testing, + ); let slots_json: Vec = matched_slots .iter() @@ -2582,6 +2645,7 @@ mod tests { gam_network_id: "21765378893".to_string(), auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, + slot: Vec::new(), } } @@ -2625,6 +2689,9 @@ mod tests { nurl: Some(nurl.to_string()), burl: Some(burl.to_string()), ad_id: Some(ad_id.to_string()), + cache_id: None, + cache_host: None, + cache_path: None, metadata: Default::default(), } } @@ -2635,11 +2702,15 @@ mod tests { let config = make_config(); let script = build_ad_slots_script(&slots, &config); assert!( - script.contains("window.__ts_ad_slots=JSON.parse"), - "should use JSON.parse" + script.contains("window.tsjs=window.tsjs||{}"), + "should initialise tsjs namespace" + ); + assert!( + script.contains(".adSlots=JSON.parse"), + "should use JSON.parse for adSlots" ); assert!(script.contains("atf_sidebar_ad"), "should include slot id"); - assert!(!script.contains("__ts_bids"), "must NOT contain bids"); + assert!(!script.contains("adInit"), "must NOT contain adInit"); assert!( !script.contains("__ts_request_id"), "must NOT contain request_id" @@ -2672,7 +2743,7 @@ mod tests { "https://ssp/bill", ), ); - let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); let entry = map.get("atf_sidebar_ad").expect("should have bid entry"); let obj = entry.as_object().expect("should be object"); assert_eq!( @@ -2688,7 +2759,7 @@ mod tests { assert_eq!( obj.get("hb_adid").and_then(|v| v.as_str()), Some("abc123"), - "should include ad_id" + "should fall back to ad_id when no cache_id present" ); assert_eq!( obj.get("nurl").and_then(|v| v.as_str()), @@ -2702,6 +2773,250 @@ mod tests { ); } + #[test] + fn client_bid_map_omits_adm_by_default() { + let mut winning_bids = HashMap::new(); + let mut bid = make_bid( + "atf_sidebar_ad", + 1.50, + "kargo", + "abc123", + "https://ssp/win", + "https://ssp/bill", + ); + bid.creative = Some("
Creative
".to_string()); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + + assert!( + obj.get("adm").is_none(), + "should omit adm when debug injection is disabled" + ); + assert!( + obj.get("debug_bid").is_none(), + "should omit debug bid when debug injection is disabled" + ); + } + + #[test] + fn client_bid_map_includes_adm_when_debug_injection_enabled() { + let mut winning_bids = HashMap::new(); + let mut bid = make_bid( + "atf_sidebar_ad", + 1.50, + "kargo", + "abc123", + "https://ssp/win", + "https://ssp/bill", + ); + bid.creative = Some("
Creative
".to_string()); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, true); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + + assert_eq!( + obj.get("adm").and_then(|v| v.as_str()), + Some("
Creative
"), + "should include adm when debug injection is enabled" + ); + } + + #[test] + fn client_bid_map_includes_debug_bid_when_debug_injection_enabled() { + let mut winning_bids = HashMap::new(); + let mut bid = make_bid( + "atf_sidebar_ad", + 1.50, + "mocktioneer", + "bid-ad-id", + "https://ssp/win", + "https://ssp/bill", + ); + bid.creative = Some("
Creative
".to_string()); + bid.adomain = Some(vec!["example.com".to_string()]); + bid.cache_id = Some("cache-uuid".to_string()); + bid.cache_host = Some("cache.example".to_string()); + bid.cache_path = Some("/cache".to_string()); + bid.metadata.insert( + "raw_field".to_string(), + serde_json::Value::String("raw-value".to_string()), + ); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, true); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + let debug_bid = obj + .get("debug_bid") + .and_then(|v| v.as_object()) + .expect("should include debug bid when debug injection is enabled"); + + assert_eq!( + debug_bid.get("slot_id").and_then(|v| v.as_str()), + Some("atf_sidebar_ad"), + "should expose original slot id" + ); + assert_eq!( + debug_bid.get("bidder").and_then(|v| v.as_str()), + Some("mocktioneer"), + "should expose original bidder" + ); + assert_eq!( + debug_bid.get("ad_id").and_then(|v| v.as_str()), + Some("bid-ad-id"), + "should expose original bid ad id" + ); + assert_eq!( + debug_bid.get("cache_id").and_then(|v| v.as_str()), + Some("cache-uuid"), + "should expose original PBS cache id" + ); + assert_eq!( + debug_bid.get("metadata").and_then(|v| v.get("raw_field")), + Some(&serde_json::Value::String("raw-value".to_string())), + "should expose provider metadata" + ); + } + + #[test] + fn bid_map_uses_cache_id_for_hb_adid_when_present() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "thetradedesk".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("bid-impression-id".to_string()), + cache_id: Some("f47447a0-b759-4f2f-9887-af458b79b570".to_string()), + cache_host: Some("openads.adsrvr.org".to_string()), + cache_path: Some("/cache".to_string()), + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should use cache_id for hb_adid, not ad_id" + ); + assert_eq!( + obj.get("hb_cache_host").and_then(|v| v.as_str()), + Some("openads.adsrvr.org"), + "should emit hb_cache_host" + ); + assert_eq!( + obj.get("hb_cache_path").and_then(|v| v.as_str()), + Some("/cache"), + "should emit hb_cache_path" + ); + } + + #[test] + fn bid_map_falls_back_to_ad_id_when_cache_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "amazon-aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("aps-bid-token".to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("aps-bid-token"), + "should fall back to ad_id when cache_id absent" + ); + assert!( + obj.get("hb_cache_host").is_none(), + "should not emit hb_cache_host when absent" + ); + assert!( + obj.get("hb_cache_path").is_none(), + "should not emit hb_cache_path when absent" + ); + } + + #[test] + fn bid_map_omits_hb_adid_when_both_cache_id_and_ad_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "amazon-aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + assert!( + obj.get("hb_adid").is_none(), + "should omit hb_adid when no cache_id and no ad_id" + ); + } + #[test] fn bid_map_excludes_slot_when_price_is_none() { let mut winning_bids = HashMap::new(); @@ -2719,10 +3034,13 @@ mod tests { nurl: None, burl: None, ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: Default::default(), }, ); - let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); assert!( map.is_empty(), "slot with no price should be excluded from bid map" @@ -2789,9 +3107,7 @@ mod tests { mod page_bids_no_match_tests { use super::super::*; use crate::auction::AuctionOrchestrator; - use crate::creative_opportunities::{ - CreativeOpportunitiesFile, CreativeOpportunityFormat, CreativeOpportunitySlot, - }; + use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; use crate::platform::test_support::noop_services; use crate::test_support::tests::crate_test_settings_str; use fastly::http::Method; @@ -2805,24 +3121,22 @@ mod tests { Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } - fn file_with_article_slot() -> CreativeOpportunitiesFile { - CreativeOpportunitiesFile { - slots: vec![CreativeOpportunitySlot { - id: "atf".to_string(), - gam_unit_path: None, - div_id: None, - page_patterns: vec!["/20**".to_string()], - formats: vec![CreativeOpportunityFormat { - width: 300, - height: 250, - media_type: crate::auction::types::MediaType::Banner, - }], - floor_price: Some(0.50), - targeting: Default::default(), - providers: Default::default(), - compiled_patterns: Vec::new(), + fn article_slot() -> Vec { + vec![CreativeOpportunitySlot { + id: "atf".to_string(), + gam_unit_path: None, + div_id: None, + page_patterns: vec!["/20**".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: crate::auction::types::MediaType::Banner, }], - } + floor_price: Some(0.50), + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + }] } fn make_page_bids_request(path: &str) -> Request { @@ -2839,10 +3153,9 @@ mod tests { let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let slots_file = CreativeOpportunitiesFile { slots: vec![] }; let req = make_page_bids_request("/2024/01/my-article/"); - let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + let response = handle_page_bids(&settings, &orchestrator, &services, &[], req) .await .expect("should return ok response"); @@ -2855,7 +3168,7 @@ mod tests { .expect("slots should be array") .len(), 0, - "empty slots file should produce zero injected slots" + "empty slots should produce zero injected slots" ); assert_eq!( body["bids"] @@ -2863,7 +3176,7 @@ mod tests { .expect("bids should be object") .len(), 0, - "empty slots file should produce zero bids" + "empty slots should produce zero bids" ); } @@ -2875,11 +3188,11 @@ mod tests { let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let slots_file = file_with_article_slot(); + let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1)"); - let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) .await .expect("should return ok response"); @@ -2911,11 +3224,11 @@ mod tests { let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let slots_file = file_with_article_slot(); + let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("sec-purpose", "prefetch"); - let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) .await .expect("should return ok response"); @@ -2946,10 +3259,10 @@ mod tests { let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let slots_file = file_with_article_slot(); // slot matches /20** only + let slots = article_slot(); // slot matches /20** only let req = make_page_bids_request("/about"); // does not match - let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) .await .expect("should return ok response"); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 386f0d54b..b221e0eac 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -416,6 +416,15 @@ pub struct DebugConfig { /// Never enable in production — visible in page source. #[serde(default)] pub auction_html_comment: bool, + + /// Include raw `adm` creative markup in `window.tsjs.bids` for GPT/GAM + /// debug rendering through the Prebid Universal Creative bridge. + /// + /// Use this to validate the server-side auction→GAM targeting→creative + /// rendering pipeline while PBS Cache is unavailable. Never enable in + /// production — injects raw HTML from SSPs. + #[serde(default)] + pub inject_adm_for_testing: bool, } #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] @@ -522,14 +531,29 @@ impl Settings { /// # Errors /// /// Returns a configuration error if any cached runtime artifact cannot be prepared. - pub fn prepare_runtime(&self) -> Result<(), Report> { + pub fn prepare_runtime(&mut self) -> Result<(), Report> { for handler in &self.handlers { handler.prepare_runtime()?; } + if let Some(co) = &mut self.creative_opportunities { + co.compile_slots(); + } + Ok(()) } + /// Returns compiled creative opportunity slots, or empty slice if feature is disabled. + #[must_use] + pub fn creative_opportunity_slots( + &self, + ) -> &[crate::creative_opportunities::CreativeOpportunitySlot] { + self.creative_opportunities + .as_ref() + .map(|co| co.slot.as_slice()) + .unwrap_or(&[]) + } + /// Resolve the first handler whose regex matches the request path. /// /// # Errors diff --git a/creative-opportunities.toml b/creative-opportunities.toml deleted file mode 100644 index da1ed23e7..000000000 --- a/creative-opportunities.toml +++ /dev/null @@ -1,47 +0,0 @@ -# Slot templates for server-side ad auction. -# Empty file = feature disabled (no auction fired, no globals injected). - -[[slot]] -id = "atf_sidebar_ad" -gam_unit_path = "/a/b/news" -div_id = "ad-atf_sidebar-0-_r_2_" -page_patterns = ["/20**", "/news/**"] -formats = [{ width = 300, height = 250 }] -floor_price = 0.50 - -[slot.targeting] -pos = "atf" -zone = "atfSidebar" - -[slot.providers.aps] -slot_id = "aps-slot-atf-sidebar" - -[[slot]] -id = "homepage_header_ad" -gam_unit_path = "/a/b/homepage" -div_id = "ad-header-0-_R_jpalubtak5lb_" -page_patterns = ["/"] -formats = [{ width = 970, height = 90 }, { width = 728, height = 90 }, { width = 970, height = 250 }] -floor_price = 0.50 - -[slot.targeting] -pos = "atf" -zone = "header" - -[slot.providers.aps] -slot_id = "aps-slot-homepage-header" - -[[slot]] -id = "homepage_footer_ad" -gam_unit_path = "/a/b/homepage" -div_id = "ad-fixed_bottom-0-_R_klubtak5lb_" -page_patterns = ["/"] -formats = [{ width = 728, height = 90 }] -floor_price = 0.50 - -[slot.targeting] -pos = "btf" -zone = "fixedBottom" - -[slot.providers.aps] -slot_id = "aps-slot-homepage-footer" diff --git a/docs/superpowers/plans/2026-05-29-pr680-reviewer-findings.md b/docs/superpowers/plans/2026-05-29-pr680-reviewer-findings.md new file mode 100644 index 000000000..83866e3b8 --- /dev/null +++ b/docs/superpowers/plans/2026-05-29-pr680-reviewer-findings.md @@ -0,0 +1,630 @@ +# PR #680 Reviewer Findings Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Address the two reviewer-required findings from PR #680 plus low-effort cleanups: consolidate slot config into `trusted-server.toml`, consolidate `window.__ts*` globals under `window.tsjs`, and fix the TypeScript `formats` type cast and `ts_initial` hardcoded string. + +**Architecture:** Slot templates move from the standalone `creative-opportunities.toml` (embedded via `include_str!`) into the `[creative_opportunities]` section of `trusted-server.toml`, using the existing `vec_from_seq_or_map` deserializer pattern already used for `BID_PARAM_ZONE_OVERRIDES`. The window globals rename is a coordinated change across `gpt_bootstrap.js`, `index.ts`, and `publisher.rs` — all three must change together since they share a runtime contract. + +**Tech Stack:** Rust (serde, toml), TypeScript, vanilla JS, `cargo test --workspace`, `npx vitest run` + +--- + +## Context for all tasks + +- **Branch:** create `fix/pr680-review-findings` off `server-side-ad-templates-impl` before starting +- **Current codebase:** `crates/trusted-server-core/`, `crates/trusted-server-adapter-fastly/`, `crates/js/lib/` +- **CI gates:** `cargo fmt`, `cargo clippy --workspace --all-targets --all-features -- -D warnings`, `cargo test --workspace`, `npx vitest run`, `npm run format` +- **Error handling:** use `error-stack` (`Report`), not anyhow. Use `derive_more::Display`, not thiserror. +- **No `unwrap()` in production code** — use `expect("should ...")`. +- **Do not** add `println!` / `eprintln!` — use `log::` macros. + +--- + +## Task 1: Consolidate slot config into `trusted-server.toml` + +**What:** Delete `creative-opportunities.toml`. Move `[[slot]]` arrays into `trusted-server.toml` as `[[creative_opportunities.slot]]`. Wire the `vec_from_seq_or_map` deserializer so env var JSON blobs also work. Remove the `SLOTS_FILE` static and `include_str!` from `main.rs`. Update `build.rs` to validate slot IDs from settings instead of a separate file. + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/settings.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-core/build.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` (function signatures) +- Modify: `trusted-server.toml` +- Delete: `creative-opportunities.toml` + +**Steps:** + +- [ ] **Step 1: Create the branch** + +```bash +git checkout -b fix/pr680-review-findings +``` + +- [ ] **Step 2: Add `Serialize` and `slot` field to structs** + +In `crates/trusted-server-core/src/creative_opportunities.rs`: + +1. Add `Serialize` to `CreativeOpportunitySlot` derive — it already has `#[serde(skip, default)]` on `compiled_patterns` so that field won't serialize. + +```rust +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CreativeOpportunitySlot { ... } +``` + +Also add `Serialize` to `CreativeOpportunityFormat`, `SlotProviders`, `ApsSlotParams` (any struct used inside `CreativeOpportunitySlot`). + +2. Add a `slot` field to `CreativeOpportunitiesConfig`: + +```rust +use crate::settings::vec_from_seq_or_map; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CreativeOpportunitiesConfig { + pub gam_network_id: String, + #[serde(default)] + pub auction_timeout_ms: Option, + #[serde(default = "PriceGranularity::dense")] + pub price_granularity: PriceGranularity, + /// Slot templates. Empty = feature disabled. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub slot: Vec, +} +``` + +Note: the field is named `slot` (not `slots`) to match the TOML key `[[creative_opportunities.slot]]`. + +- [ ] **Step 3: Delete `CreativeOpportunitiesFile`** + +Remove the `CreativeOpportunitiesFile` struct and its `impl` from `creative_opportunities.rs`. The `compile` logic moves to a free function or into `CreativeOpportunitiesConfig`: + +```rust +impl CreativeOpportunitiesConfig { + /// Pre-compile glob patterns for all slots. Call once after deserialization. + pub fn compile_slots(&mut self) { + for slot in &mut self.slot { + slot.compile_patterns(); + } + } +} +``` + +- [ ] **Step 4: Wire slot compilation into `Settings::prepare_runtime`** + +Glob pattern pre-compilation must happen once at startup, not per-request. `Settings::prepare_runtime` is already called after deserialization in both `from_toml_and_env` (build time) and `get_settings()` (runtime). Add slot compilation there: + +```rust +// In settings.rs, inside Settings::prepare_runtime +pub fn prepare_runtime(&mut self) -> Result<(), Report> { + for handler in &self.handlers { + handler.prepare_runtime()?; + } + // Pre-compile slot glob patterns for hot-path matching. + if let Some(co) = &mut self.creative_opportunities { + co.compile_slots(); + } + Ok(()) +} +``` + +Note: `prepare_runtime` must take `&mut self` for this change. Check current signature — if it takes `&self`, change it to `&mut self` and update call sites. + +Also add a helper method for call sites that need the slot slice: + +```rust +impl Settings { + /// Returns compiled creative opportunity slots, or empty slice if disabled. + pub fn creative_opportunity_slots(&self) -> &[CreativeOpportunitySlot] { + self.creative_opportunities + .as_ref() + .map(|co| co.slot.as_slice()) + .unwrap_or(&[]) + } +} +``` + +- [ ] **Step 5: Update `build.rs` stub and slot validation** + +First update the `creative_opportunities` stub in `build.rs` to add the `slot` field — without this the settings parse will fail at build time when `trusted-server.toml` contains `[[creative_opportunities.slot]]` entries: + +```rust +mod creative_opportunities { + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Clone, Deserialize, Serialize)] + pub struct CreativeOpportunitiesConfig { + pub gam_network_id: String, + #[serde(default)] + pub auction_timeout_ms: Option, + #[serde(default = "default_price_granularity")] + pub price_granularity: String, + // Use serde_json::Value to avoid pulling in full slot type in build context. + #[serde(default)] + pub slot: Vec, + } + + fn default_price_granularity() -> String { + "dense".to_string() + } +} +``` + +Then replace the separate-file validation block with reading slots from `Settings`: + +```rust +// After settings are parsed, validate slot IDs +let slot_id_re = regex::Regex::new(r"^[A-Za-z0-9_\-]+$").expect("should compile regex"); +if let Some(co) = &settings.creative_opportunities { + for slot in &co.slot { + if let Err(e) = trusted_server_core::creative_opportunities::validate_slot_id(&slot.id) { + panic!("trusted-server.toml [creative_opportunities.slot]: {e}"); + } + } + if !co.slot.is_empty() { + println!( + "cargo:warning=creative_opportunities: {} slot(s) validated", + co.slot.len() + ); + } +} +``` + +Remove: `CREATIVE_OPPORTUNITIES_PATH` const, the `co_path.exists()` block, and the `println!("cargo:rerun-if-changed={}", CREATIVE_OPPORTUNITIES_PATH)` line. + +Note: `build.rs` already pulls in `src/creative_opportunities.rs` as a module — make sure the module stub includes the new `Serialize` derive (it may need the serde `Serialize` import). + +- [ ] **Step 6: Update `main.rs` — remove `SLOTS_FILE` static** + +Remove: + +```rust +const CREATIVE_OPPORTUNITIES_TOML: &str = include_str!("../../../creative-opportunities.toml"); +static SLOTS_FILE: std::sync::LazyLock<...> = ...; +``` + +Replace `slots_file` parameter threading with deriving slots from `settings`: + +Where `slots_file` was passed as `&*SLOTS_FILE`, pass `settings.creative_opportunity_slots()` instead. This requires `settings` to be available at that call site (it is — `settings` is already in scope). + +Update function signatures in `main.rs` that reference `CreativeOpportunitiesFile` to accept `&[CreativeOpportunitySlot]` instead. + +- [ ] **Step 7: Update `publisher.rs` function signatures** + +Functions that take `&crate::creative_opportunities::CreativeOpportunitiesFile` change to `&[crate::creative_opportunities::CreativeOpportunitySlot]`: + +```rust +// Before +pub(crate) fn handle_page_bids( + ... + slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, + ... +) + +// After +pub(crate) fn handle_page_bids( + ... + slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + ... +) +``` + +Inside the function body, replace `slots_file.slots` with `slots`. + +Update all call sites and test helpers in `publisher.rs` that construct `CreativeOpportunitiesFile { slots: vec![...] }` to pass `&[slot]` directly. + +- [ ] **Step 8: Update `trusted-server.toml`** + +Move the slots from `creative-opportunities.toml` into `trusted-server.toml` under `[creative_opportunities]`. Use `[[creative_opportunities.slot]]` syntax. Use only example/fictional values per project convention (example.com domains, fictional IDs): + +```toml +[creative_opportunities] +gam_network_id = "88059007" +auction_timeout_ms = 1500 +price_granularity = "dense" + +[[creative_opportunities.slot]] +id = "atf_sidebar_ad" +gam_unit_path = "/a/b/news" +div_id = "div-ad-atf-sidebar" +page_patterns = ["/news/**"] +formats = [{ width = 300, height = 250 }] +floor_price = 0.50 + +[creative_opportunities.slot.targeting] +pos = "atf" +zone = "atfSidebar" + +[creative_opportunities.slot.providers.aps] +slot_id = "aps-slot-atf-sidebar" +``` + +- [ ] **Step 9: Delete `creative-opportunities.toml`** + +```bash +git rm creative-opportunities.toml +``` + +- [ ] **Step 10: Run tests** + +```bash +cargo test --workspace +``` + +Expected: all tests pass. Fix any compile errors from the signature changes. + +- [ ] **Step 11: Run clippy and fmt** + +```bash +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +``` + +- [ ] **Step 12: Commit** + +```bash +git add -p +git commit -m "Move slot templates from creative-opportunities.toml into trusted-server.toml" +``` + +--- + +## Task 2: Consolidate `window.__ts*` globals under `window.tsjs` + +**What:** All `window.__ts*` globals become properties on a single `window._ts` namespace object. Changes must be coordinated across three files: `gpt_bootstrap.js`, `index.ts`, and `publisher.rs`. Tests in `index.test.ts` must be updated too. + +**Rename table:** + +| Old global | New property | Notes | +| ----------------------------- | ------------------------------ | ---------------------------- | +| `window.__ts_ad_slots` | `window.tsjs.adSlots` | Array, set at head-open | +| `window.__ts_bids` | `window.tsjs.bids` | Object, set before `` | +| `window.__tsAdInit` | `window.tsjs.adInit` | Function | +| `window.__tsPrevGptSlots` | `window.tsjs.prevGptSlots` | Array | +| `window.__tsServicesEnabled` | `window.tsjs.servicesEnabled` | Boolean | +| `window.__tsDivToSlotId` | `window.tsjs.divToSlotId` | Object | +| `window.__tsSpaHookInstalled` | `window.tsjs.spaHookInstalled` | Boolean | + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` +- Modify: `crates/js/lib/src/integrations/gpt/index.ts` +- Modify: `crates/js/lib/src/integrations/gpt/index.test.ts` +- Modify: `crates/js/lib/test/integrations/gpt/index.test.ts` (if exists) + +**Steps:** + +- [ ] **Step 1: Update `publisher.rs` injected scripts** + +`build_ad_slots_script` generates the `", escaped) + +// After — initialise _ts if absent, then set adSlots +format!("", escaped) +``` + +`build_bids_script` generates the script injected before ``. Change: + +```rust +// Before +format!( + "", + escaped +) + +// After +format!( + "", + escaped +) +``` + +Note: `{{}}` is the Rust format-string escape for a literal `{}`. + +Update any test assertions in `publisher.rs` that check for the old global names. + +- [ ] **Step 2: Update `gpt_bootstrap.js`** + +Replace all `window.__ts*` references. The bootstrap IIFE runs before the TS bundle, so it must initialise `window._ts` if absent: + +```js +;(function () { + if (typeof window === 'undefined') return + // Initialise namespace; adInit guard prevents double-install. + var ts = (window._ts = window._ts || {}) + if (ts.adInit) return + + ts.adInit = function () { + var slots = ts.adSlots || [] + var bids = ts.bids || {} + var divToSlotId = {} + googletag.cmd.push(function () { + var newSlots = [] + slots.forEach(function (slot) { + var s = googletag.defineSlot( + slot.gam_unit_path, + slot.formats, + slot.div_id + ) + if (!s) return + s.addService(googletag.pubads()) + Object.entries(slot.targeting || {}).forEach(function (e) { + s.setTargeting(e[0], e[1]) + }) + var b = bids[slot.id] || {} + ;['hb_pb', 'hb_bidder', 'hb_adid'].forEach(function (k) { + if (b[k]) s.setTargeting(k, b[k]) + }) + s.setTargeting('ts_initial', '1') + divToSlotId[slot.div_id] = slot.id + newSlots.push(s) + }) + ts.prevGptSlots = newSlots + ts.divToSlotId = divToSlotId + if (!ts.servicesEnabled) { + googletag.pubads().enableSingleRequest() + googletag.enableServices() + ts.servicesEnabled = true + googletag.pubads().addEventListener('slotRenderEnded', function (ev) { + var divId = ev.slot.getSlotElementId() + var slotId = (ts.divToSlotId || {})[divId] + if (!slotId) return + var b = (ts.bids || {})[slotId] || {} + var ourBidWon = + !ev.isEmpty && + (b.hb_adid + ? ev.slot.getTargeting('hb_adid')[0] === b.hb_adid + : !!b.hb_bidder) + if (ourBidWon) { + if (b.nurl) navigator.sendBeacon(b.nurl) + if (b.burl) navigator.sendBeacon(b.burl) + } + }) + } + if (newSlots.length > 0) { + googletag.pubads().refresh(newSlots) + } + }) + } +})() +``` + +- [ ] **Step 3: Update `index.ts` — rename `TsWindow` type** + +Replace the `TsWindow` interface: + +```typescript +type TsNamespace = { + adSlots?: TsAdSlot[] + bids?: Record + adInit?: () => void + prevGptSlots?: GoogleTagSlot[] + servicesEnabled?: boolean + divToSlotId?: Record + spaHookInstalled?: boolean +} + +type TsWindow = Window & { + _ts?: TsNamespace +} +``` + +- [ ] **Step 4: Update `installTsAdInit` in `index.ts`** + +Update all properties to live under `window.tsjs`. Use `window.tsjs` directly: + +```typescript +export function installTsAdInit(): void { + const w = window as TsWindow + const ts = (w._ts = w._ts ?? {}) + ts.adInit = function () { + const slots = ts.adSlots ?? [] + const bids = ts.bids ?? {} + const g = (window as GptWindow).googletag + if (!g) return + + g.cmd?.push(() => { + if (ts.prevGptSlots && ts.prevGptSlots.length > 0) { + g.destroySlots?.(ts.prevGptSlots) + ts.prevGptSlots = [] + } + const newSlots: GoogleTagSlot[] = [] + const divToSlotId: Record = {} + + slots.forEach((slot) => { + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats as Array, + slot.div_id + ) + if (!gptSlot) return + gptSlot.addService(g.pubads!()) + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => + gptSlot.setTargeting(k, v) + ) + const bid = bids[slot.id] ?? {} + ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + }) + gptSlot.setTargeting('ts_initial', '1') + divToSlotId[slot.div_id] = slot.id + newSlots.push(gptSlot) + }) + + ts.prevGptSlots = newSlots + ts.divToSlotId = divToSlotId + + if (!ts.servicesEnabled) { + g.pubads!().enableSingleRequest() + g.enableServices?.() + ts.servicesEnabled = true + g.pubads!().addEventListener?.( + 'slotRenderEnded', + (event: SlotRenderEndedEvent) => { + const divId: string = event.slot?.getSlotElementId?.() ?? '' + const slotId = (ts.divToSlotId ?? {})[divId] + if (!slotId) return + const bid = (ts.bids ?? {})[slotId] ?? {} + const ourBidWon = + !event.isEmpty && + (bid.hb_adid + ? event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + : !!bid.hb_bidder) + if (ourBidWon) { + if (bid.nurl) navigator.sendBeacon(bid.nurl) + if (bid.burl) navigator.sendBeacon(bid.burl) + } + } + ) + } + if (newSlots.length > 0) { + g.pubads!().refresh(newSlots) + } + }) + } +} +``` + +- [ ] **Step 5: Update `installSpaHook` in `index.ts`** + +Replace `__tsSpaHookInstalled` and `__ts_ad_slots`/`__ts_bids` reads: + +```typescript +export function installSpaHook(): void { + const win = window as TsWindow + const ts = (win._ts = win._ts ?? {}) + if (ts.spaHookInstalled) return + ts.spaHookInstalled = true + // ... rest of SPA hook logic uses ts.adSlots, ts.bids, ts.adInit +} +``` + +- [ ] **Step 6: Update tests in `index.test.ts`** + +Find all test assertions that reference `window.__ts_ad_slots`, `window.__ts_bids`, `window.__tsAdInit`, etc. and update to `window.tsjs.adSlots`, `window.tsjs.bids`, `window.tsjs.adInit` etc. + +Run tests first to see what fails: + +```bash +cd crates/js/lib && npx vitest run +``` + +Fix each failing assertion. + +- [ ] **Step 7: Run JS tests and format** + +```bash +cd crates/js/lib && npx vitest run +cd crates/js/lib && npm run format +``` + +Expected: all tests pass, no format errors. + +- [ ] **Step 8: Run Rust tests** + +```bash +cargo test --workspace +``` + +Update any test assertions in `publisher.rs` that check for old global names (e.g. `script.contains("window.__ts_ad_slots")`). + +- [ ] **Step 9: Run clippy and fmt** + +```bash +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +``` + +- [ ] **Step 10: Commit** + +```bash +git commit -m "Namespace window globals under window._ts" +``` + +--- + +## Task 3: Fix `formats` type and extract `ts_initial` constant + +**What:** Two small TypeScript/JS cleanups. `TsAdSlot.formats` should be typed as `Array<[number, number]>` (tuple, not array-of-array) to match GPT's actual input. The string `'ts_initial'` is hardcoded in both `gpt_bootstrap.js` and `index.ts` — extract as a named constant in `index.ts` (no JS equivalent needed since the bootstrap is vanilla JS). + +**Files:** + +- Modify: `crates/js/lib/src/integrations/gpt/index.ts` +- Modify: `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` (comment only — JS can't share TS constants) + +**Steps:** + +- [ ] **Step 1: Fix `TsAdSlot.formats` type** + +In `index.ts`, change: + +```typescript +// Before +interface TsAdSlot { + ... + formats: Array; +} + +// After +interface TsAdSlot { + ... + formats: Array<[number, number]>; +} +``` + +Update the cast at the GPT `defineSlot` call site — `[number, number]` satisfies `number | number[]` so the cast can be removed or simplified: + +```typescript +// Before +slot.formats as Array + +// After — [number, number][] already satisfies Array +slot.formats +``` + +- [ ] **Step 2: Extract `ts_initial` constant in `index.ts`** + +Near the top of `index.ts`, add: + +```typescript +const TS_INITIAL_TARGETING_KEY = 'ts_initial' +``` + +Replace both occurrences of `'ts_initial'` in `installTsAdInit` with `TS_INITIAL_TARGETING_KEY`. + +Add a comment in `gpt_bootstrap.js` where `'ts_initial'` appears: + +```js +// Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts +s.setTargeting('ts_initial', '1') +``` + +- [ ] **Step 3: Run JS tests and format** + +```bash +cd crates/js/lib && npx vitest run +cd crates/js/lib && npm run format +``` + +- [ ] **Step 4: Commit** + +```bash +git commit -m "Fix TsAdSlot formats type and extract ts_initial constant" +``` + +--- + +## Final verification + +- [ ] `cargo fmt --all -- --check` +- [ ] `cargo clippy --workspace --all-targets --all-features -- -D warnings` +- [ ] `cargo test --workspace` +- [ ] `cd crates/js/lib && npx vitest run` +- [ ] `cd crates/js/lib && npm run format` +- [ ] `cd docs && npm run format` diff --git a/docs/superpowers/plans/2026-05-29-prebid-creative-rendering-fix.md b/docs/superpowers/plans/2026-05-29-prebid-creative-rendering-fix.md new file mode 100644 index 000000000..7a3f34207 --- /dev/null +++ b/docs/superpowers/plans/2026-05-29-prebid-creative-rendering-fix.md @@ -0,0 +1,760 @@ +# Prebid Creative Rendering Fix Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix `hb_adid` to carry the PBS Cache UUID (not the OpenRTB bid ID) so the Prebid Universal Creative in GAM can fetch and render the correct creative markup. + +**Architecture:** Three-file change: add `cache_id`/`cache_host`/`cache_path` fields to the shared `Bid` struct in `types.rs`, extract these from `ext.prebid.cache.bids` in `prebid.rs`'s `parse_bid`, then emit them as `hb_adid`/`hb_cache_host`/`hb_cache_path` in `publisher.rs`'s `build_bid_map`. `AuctionBid` in `prebid.rs` is a type alias for `Bid` (`use ... Bid as AuctionBid`), so only one struct needs the new fields. + +**Tech Stack:** Rust 2024, `serde`, `url` crate (already in workspace deps at v2.5.8), `cargo test --workspace` + +--- + +## Context for all tasks + +- **Branch:** `fix/server-side-ad-template-entrypoint` (already checked out) +- **Spec:** `docs/superpowers/specs/2026-05-29-prebid-creative-rendering-fix.md` +- **Error handling:** `error-stack` (`Report`), not anyhow. Use `expect("should ...")` not `unwrap()`. +- **No `println!`/`eprintln!`** — use `log::` macros. +- **All public items must have doc comments.** +- CI gates: `cargo fmt --all -- --check`, `cargo clippy --workspace --all-targets --all-features -- -D warnings`, `cargo test --workspace` + +--- + +## Task 1: Add cache fields to `Bid` struct and fix all construction sites + +**What:** Add three new `Option` fields to `Bid`. Since Rust struct literals are exhaustive, every place that constructs a `Bid { ... }` in the codebase will fail to compile until the new fields are added. Fix all of them with `None` defaults (except the APS provider which constructs a real `Bid` — also `None` since APS doesn't use PBS Cache). + +**Files:** + +- Modify: `crates/trusted-server-core/src/auction/types.rs:200` (after `ad_id` field) +- Modify (test helpers/literals — add `None` fields): + - `crates/trusted-server-core/src/auction/types.rs:314` (`make_bid` helper) + - `crates/trusted-server-core/src/auction/types.rs:445` (inline `Bid` literal) + - `crates/trusted-server-core/src/publisher.rs:2616` (`make_bid` helper) + - `crates/trusted-server-core/src/publisher.rs:2714` (inline `Bid` literal) + - `crates/trusted-server-core/src/auction/orchestrator.rs:1121,1138,1278,1325,1358` (test `Bid` literals) + - `crates/trusted-server-core/src/integrations/aps.rs:442` (production `Bid` construction) + +**Steps:** + +- [ ] **Step 1: Add three fields to `Bid` struct in `types.rs`** + + In `crates/trusted-server-core/src/auction/types.rs`, after line 200 (`pub ad_id: Option,`), add: + + ```rust + /// Prebid Cache UUID for this bid. + /// + /// Populated from `ext.prebid.cache.bids.cacheId` in the PBS response. + /// Used as `hb_adid` targeting value in `window._ts.bids`. `None` for + /// non-PBS providers (e.g., APS) and PBS bids without Prebid Cache enabled. + pub cache_id: Option, + /// Prebid Cache host (e.g., `"openads.adsrvr.org"`). + /// + /// Populated from the host of `ext.prebid.cache.bids.url`. Used as + /// `hb_cache_host` targeting value. `None` when cache is absent. + pub cache_host: Option, + /// Prebid Cache path (e.g., `"/cache"`). + /// + /// Populated from the path of `ext.prebid.cache.bids.url`. Used as + /// `hb_cache_path` targeting value. `None` when cache is absent. + pub cache_path: Option, + ``` + +- [ ] **Step 2: Verify compile fails as expected** + + ```bash + cargo check --package trusted-server-core 2>&1 | grep "missing field" + ``` + + Expected: multiple errors about missing `cache_id`, `cache_host`, `cache_path` in `Bid` struct literals. This confirms every construction site will be found. + +- [ ] **Step 3: Fix `make_bid` helper in `types.rs` (line ~314)** + + Add three `None` fields to the `Bid {}` literal inside the `make_bid` test helper: + + ```rust + fn make_bid(bidder: &str) -> Bid { + Bid { + slot_id: "slot-1".to_string(), + price: Some(1.0), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: bidder.to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + } + } + ``` + +- [ ] **Step 4: Fix inline `Bid` literal in `types.rs` (line ~445)** + + Find the `Bid {` literal around line 445 in the test section of `types.rs`. Add: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 5: Fix `make_bid` helper in `publisher.rs` (line ~2616)** + + In the `make_bid` test helper function in `publisher.rs`, add to the `Bid {}` literal: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 6: Fix inline `Bid` literal in `publisher.rs` (line ~2714)** + + Find the `Bid {` literal around line 2714 in `publisher.rs` tests. Add: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 7: Fix five `Bid` literals in `orchestrator.rs` (lines ~1121,1138,1278,1325,1358)** + + Add to each of the five `Bid {}` literals in the test section of `orchestrator.rs`: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 8: Fix APS production `Bid` construction in `aps.rs` (line ~442)** + + In `aps.rs`, inside `parse_aps_response` (or wherever the `Ok(Bid { ... })` is around line 442), add: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + + APS does not use PBS Cache — these fields are intentionally `None` for APS bids. + +- [ ] **Step 9: Verify compile succeeds** + + ```bash + cargo check --package trusted-server-core 2>&1 | grep -E "^error" + ``` + + Expected: no output (clean compile). + +- [ ] **Step 10: Run tests to confirm nothing regressed** + + ```bash + cargo test --workspace 2>&1 | tail -5 + ``` + + Expected: all tests pass. + +- [ ] **Step 11: Run clippy and fmt** + + ```bash + cargo fmt --all + cargo clippy --workspace --all-targets --all-features -- -D warnings 2>&1 | tail -5 + ``` + + Expected: clean. + +- [ ] **Step 12: Commit** + + ```bash + git add crates/trusted-server-core/src/auction/types.rs \ + crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-core/src/auction/orchestrator.rs \ + crates/trusted-server-core/src/integrations/aps.rs + git commit -m "Add cache_id, cache_host, cache_path fields to Bid struct" + ``` + +--- + +## Task 2: Extract PBS Cache fields in `prebid.rs` `parse_bid` + tests + +**What:** After extracting `ad_id` in `parse_bid`, extract `ext.prebid.cache.bids.cacheId` as `cache_id` and split `ext.prebid.cache.bids.url` into `cache_host` + `cache_path`. Populate all three new fields on the returned `AuctionBid`. Add TDD tests first. + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/prebid.rs:1362–1391` (extraction + struct literal) +- Test: `crates/trusted-server-core/src/integrations/prebid.rs` (test module near bottom) + +**Steps:** + +- [ ] **Step 1: Write the failing tests** + + Find the `#[cfg(test)]` module in `prebid.rs`. Add these tests (they will fail because extraction doesn't exist yet): + + ```rust + #[test] + fn parse_bid_extracts_cache_id_from_ext_prebid_cache_bids() { + // Real PBS response shape from auction_response.json + let bid_json = serde_json::json!({ + "id": "bid-id-123", + "impid": "atf_sidebar_ad", + "price": 1.50, + "adm": "
ad
", + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://openads.adsrvr.org/cache?uuid=f47447a0-b759-4f2f-9887-af458b79b570", + "cacheId": "f47447a0-b759-4f2f-9887-af458b79b570" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "thetradedesk") + .expect("should parse bid"); + assert_eq!( + bid.cache_id.as_deref(), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should extract cacheId as cache_id" + ); + assert_eq!( + bid.cache_host.as_deref(), + Some("openads.adsrvr.org"), + "should extract host from cache URL" + ); + assert_eq!( + bid.cache_path.as_deref(), + Some("/cache"), + "should extract path from cache URL" + ); + } + + #[test] + fn parse_bid_sets_cache_fields_to_none_when_no_cache_entry() { + let bid_json = serde_json::json!({ + "id": "bid-id-456", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250 + // no ext.prebid.cache + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert!(bid.cache_id.is_none(), "should be None when cache absent"); + assert!(bid.cache_host.is_none(), "should be None when cache absent"); + assert!(bid.cache_path.is_none(), "should be None when cache absent"); + } + + #[test] + fn parse_bid_handles_malformed_cache_url_gracefully() { + let bid_json = serde_json::json!({ + "id": "bid-id-789", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "not-a-valid-url", + "cacheId": "some-uuid" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid without panicking"); + assert_eq!( + bid.cache_id.as_deref(), + Some("some-uuid"), + "should still extract cacheId even if URL is malformed" + ); + assert!(bid.cache_host.is_none(), "should be None when URL parse fails"); + assert!(bid.cache_path.is_none(), "should be None when URL parse fails"); + } + + #[test] + fn parse_bid_preserves_ad_id_alongside_cache_id() { + let bid_json = serde_json::json!({ + "id": "bid-impression-id", + "impid": "atf_sidebar_ad", + "adid": "bidder-ad-id-abc", + "price": 1.0, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://cache.example.com/cache", + "cacheId": "cache-uuid-xyz" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert_eq!( + bid.ad_id.as_deref(), + Some("bidder-ad-id-abc"), + "should keep ad_id from adid field" + ); + assert_eq!( + bid.cache_id.as_deref(), + Some("cache-uuid-xyz"), + "should extract cache UUID separately" + ); + } + ``` + + Note: `base_config()` and `PrebidAuctionProvider::new()` are the standard test construction pattern used throughout the existing `prebid.rs` test module. `parse_bid` is a private method but is accessible from the `#[cfg(test)]` module in the same file. + +- [ ] **Step 2: Run tests to verify they fail** + + ```bash + cargo test --package trusted-server-core parse_bid_extracts_cache_id 2>&1 | tail -15 + ``` + + Expected: compile error (`no field 'cache_id' on type 'Bid'`) or test failure. Either confirms the extraction code is missing. + +- [ ] **Step 3: Add cache extraction to `parse_bid` in `prebid.rs`** + + In `parse_bid` (around line 1362), after the `ad_id` extraction block and before the `Ok(AuctionBid { ... })`, add: + + ```rust + // Extract PBS Cache coordinates from ext.prebid.cache.bids. + // The Prebid Universal Creative uses cacheId as hb_adid and the host/path + // to construct the fetch URL: https://?uuid= + let cache_entry = bid_obj + .get("ext") + .and_then(|e| e.get("prebid")) + .and_then(|p| p.get("cache")) + .and_then(|c| c.get("bids")); + + let cache_id = cache_entry + .and_then(|c| c.get("cacheId")) + .and_then(|v| v.as_str()) + .map(String::from); + + let (cache_host, cache_path) = cache_entry + .and_then(|c| c.get("url")) + .and_then(|v| v.as_str()) + .and_then(|url_str| { + url::Url::parse(url_str) + .map_err(|e| log::debug!("PBS cache URL parse failed: {e}")) + .ok() + }) + .map(|u| { + let host = u.host_str().map(String::from); + let path = u.path().to_string(); + let path = if path.is_empty() || path == "/" { + None + } else { + Some(path) + }; + (host, path) + }) + .unwrap_or((None, None)); + + if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — \ + creative will fail to render for slot '{slot_id}'" + ); + } + ``` + + Then add the three fields to the `Ok(AuctionBid { ... })` struct literal (around line 1377): + + ```rust + Ok(AuctionBid { + slot_id, + price: Some(price), + currency: DEFAULT_CURRENCY.to_string(), + creative, + adomain, + bidder: seat.to_string(), + width, + height, + nurl, + burl, + ad_id, + cache_id, + cache_host, + cache_path, + metadata: std::collections::HashMap::new(), + }) + ``` + +- [ ] **Step 4: Run tests to verify they pass** + + ```bash + cargo test --package trusted-server-core parse_bid 2>&1 | tail -20 + ``` + + Expected: all 4 new tests pass. + +- [ ] **Step 5: Run full test suite** + + ```bash + cargo test --workspace 2>&1 | tail -5 + ``` + + Expected: all tests pass. + +- [ ] **Step 6: Run clippy and fmt** + + ```bash + cargo fmt --all + cargo clippy --workspace --all-targets --all-features -- -D warnings 2>&1 | tail -5 + ``` + + Expected: clean. If clippy warns about the `log::debug!` return value being unused inside `map_err`, suppress with `let _ = ...` or restructure. + +- [ ] **Step 7: Commit** + + ```bash + git add crates/trusted-server-core/src/integrations/prebid.rs + git commit -m "Extract PBS Cache UUID and endpoint from bid ext into Bid fields" + ``` + +--- + +## Task 3: Emit cache fields in `build_bid_map` + update tests + +**What:** Change `build_bid_map` to use `bid.cache_id` for `hb_adid` (falling back to `bid.ad_id` for APS/other providers), and emit `hb_cache_host`/`hb_cache_path` when present. Update the existing `bid_map_includes_nurl_and_burl` test (which currently passes `"abc123"` as `ad_id` and asserts `hb_adid = "abc123"`) to use a cache-based bid. Add new tests covering cache fields and fallback path. + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs:1311–1342` (`build_bid_map`) +- Modify: `crates/trusted-server-core/src/publisher.rs:2608–2630` (`make_bid` helper — add cache params) +- Modify: `crates/trusted-server-core/src/publisher.rs:2666–2707` (existing `bid_map_includes_nurl_and_burl` test) +- Test: `crates/trusted-server-core/src/publisher.rs` (new tests in the existing test module) + +**Steps:** + +- [ ] **Step 1: Write new failing tests for cache field emission** + + Add these tests to the `#[cfg(test)]` module in `publisher.rs`, near the existing `bid_map_includes_nurl_and_burl` test: + + ```rust + #[test] + fn bid_map_uses_cache_id_for_hb_adid_when_present() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "thetradedesk".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("bid-impression-id".to_string()), + cache_id: Some("f47447a0-b759-4f2f-9887-af458b79b570".to_string()), + cache_host: Some("openads.adsrvr.org".to_string()), + cache_path: Some("/cache".to_string()), + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let obj = map + .get("atf_sidebar_ad") + .expect("should have entry") + .as_object() + .expect("should be object"); + + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should use cache_id for hb_adid, not ad_id" + ); + assert_eq!( + obj.get("hb_cache_host").and_then(|v| v.as_str()), + Some("openads.adsrvr.org"), + "should emit hb_cache_host" + ); + assert_eq!( + obj.get("hb_cache_path").and_then(|v| v.as_str()), + Some("/cache"), + "should emit hb_cache_path" + ); + } + + #[test] + fn bid_map_falls_back_to_ad_id_when_cache_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "aps-amazon".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("aps-bid-token".to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let obj = map + .get("atf_sidebar_ad") + .expect("should have entry") + .as_object() + .expect("should be object"); + + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("aps-bid-token"), + "should fall back to ad_id when cache_id absent" + ); + assert!( + obj.get("hb_cache_host").is_none(), + "should not emit hb_cache_host when absent" + ); + assert!( + obj.get("hb_cache_path").is_none(), + "should not emit hb_cache_path when absent" + ); + } + + #[test] + fn bid_map_omits_hb_adid_when_both_cache_id_and_ad_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "amazon-aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let obj = map + .get("atf_sidebar_ad") + .expect("should have entry") + .as_object() + .expect("should be object"); + + assert!( + obj.get("hb_adid").is_none(), + "should omit hb_adid when no cache_id and no ad_id" + ); + } + ``` + +- [ ] **Step 2: Run tests to verify they fail** + + ```bash + cargo test --package trusted-server-core bid_map_uses_cache_id 2>&1 | tail -15 + ``` + + Expected: test fails — `hb_adid` returns `"bid-impression-id"` (the wrong value) instead of the cache UUID, and `hb_cache_host`/`hb_cache_path` are not emitted. + +- [ ] **Step 3: Update `build_bid_map` in `publisher.rs`** + + Replace the current `hb_adid` emission block (lines ~1326–1331) and the `nurl`/`burl` block with: + + ```rust + // hb_adid: PBS Cache UUID when present (Prebid Universal Creative uses this + // as the cache lookup key). Falls back to ad_id for APS and other non-PBS + // providers. Note: ad_id (OpenRTB bid ID) is NOT the same as the cache UUID. + let hb_adid = bid.cache_id.as_deref().or(bid.ad_id.as_deref()); + if let Some(id) = hb_adid { + obj.insert( + "hb_adid".to_string(), + serde_json::Value::String(id.to_string()), + ); + } + + // Cache endpoint coordinates — only present for PBS bids with Prebid Cache. + // The Prebid Universal Creative constructs: + // https://?uuid= + if let Some(ref host) = bid.cache_host { + obj.insert( + "hb_cache_host".to_string(), + serde_json::Value::String(host.clone()), + ); + } + if let Some(ref path) = bid.cache_path { + obj.insert( + "hb_cache_path".to_string(), + serde_json::Value::String(path.clone()), + ); + } + + if let Some(ref nurl) = bid.nurl { + obj.insert("nurl".to_string(), serde_json::Value::String(nurl.clone())); + } + if let Some(ref burl) = bid.burl { + obj.insert("burl".to_string(), serde_json::Value::String(burl.clone())); + } + ``` + +- [ ] **Step 4: Update the existing `bid_map_includes_nurl_and_burl` test** + + The existing test at line ~2666 constructs a bid via `make_bid("atf_sidebar_ad", 1.50, "kargo", "abc123", ...)` and asserts `hb_adid = "abc123"`. Update `make_bid` to accept optional `cache_id`, `cache_host`, `cache_path`, OR create a separate variant. The simplest fix: update the assertion in the existing test to reflect the new priority logic. + + The test currently passes `ad_id = "abc123"` and `cache_id = None`. After the fix, `hb_adid` should still be `"abc123"` (fallback path). So the existing assertion is correct — just verify it still passes. No change needed to that test body. Just update `make_bid` to set the new fields to `None`: + + ```rust + fn make_bid( + slot_id: &str, + price: f64, + bidder: &str, + ad_id: &str, + nurl: &str, + burl: &str, + ) -> Bid { + Bid { + slot_id: slot_id.to_string(), + price: Some(price), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: bidder.to_string(), + width: 300, + height: 250, + nurl: Some(nurl.to_string()), + burl: Some(burl.to_string()), + ad_id: Some(ad_id.to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + } + } + ``` + + Also update the assertion comment at line ~2694 from `"should include ad_id"` to `"should fall back to ad_id when no cache_id"`. + +- [ ] **Step 5: Run all new tests** + + ```bash + cargo test --package trusted-server-core bid_map 2>&1 | tail -20 + ``` + + Expected: all `bid_map_*` tests pass, including both new and existing. + +- [ ] **Step 6: Add round-trip serialization test for `Bid`** + + Add this test to the `#[cfg(test)]` module in `types.rs`: + + ```rust + #[test] + fn bid_with_cache_fields_round_trips_through_json() { + let bid = Bid { + slot_id: "atf".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "thetradedesk".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("bid-id".to_string()), + cache_id: Some("cache-uuid".to_string()), + cache_host: Some("cache.example.com".to_string()), + cache_path: Some("/pbc/v1/cache".to_string()), + metadata: HashMap::new(), + }; + let json = serde_json::to_string(&bid).expect("should serialize Bid"); + let restored: Bid = serde_json::from_str(&json).expect("should deserialize Bid"); + assert_eq!(restored.cache_id.as_deref(), Some("cache-uuid"), "should round-trip cache_id"); + assert_eq!(restored.cache_host.as_deref(), Some("cache.example.com"), "should round-trip cache_host"); + assert_eq!(restored.cache_path.as_deref(), Some("/pbc/v1/cache"), "should round-trip cache_path"); + } + ``` + + Run: + + ```bash + cargo test --package trusted-server-core bid_with_cache_fields_round_trips 2>&1 | tail -5 + ``` + + Expected: PASS. + +- [ ] **Step 7: Run full CI suite** + + ```bash + cargo test --workspace 2>&1 | tail -5 + cargo fmt --all -- --check + cargo clippy --workspace --all-targets --all-features -- -D warnings 2>&1 | tail -5 + ``` + + Expected: all pass, no warnings. + +- [ ] **Step 8: Commit** + + ```bash + git add crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-core/src/auction/types.rs + git commit -m "Emit hb_adid from PBS Cache UUID and add hb_cache_host/hb_cache_path to bid map" + ``` + +--- + +## Final verification + +- [ ] Run `cargo test --workspace` — all pass +- [ ] Run `cargo clippy --workspace --all-targets --all-features -- -D warnings` — clean +- [ ] Run `cargo fmt --all -- --check` — clean +- [ ] In browser devtools after deploy: `window._ts.bids` shows `hb_cache_host`, `hb_cache_path`, and `hb_adid` matching the UUID in `ext.prebid.cache.bids.cacheId` from the raw PBS response + +--- + +## Rollout reminder (from spec §8) + +1. TS: this branch deployed +2. GAM: ad ops updates Prebid line item creatives to server-side cache-fetch variant (see spec §4.6) +3. PBS: Prebid Cache already enabled (confirmed from real response) +4. Verify in devtools diff --git a/docs/superpowers/specs/2026-05-29-prebid-creative-rendering-fix.md b/docs/superpowers/specs/2026-05-29-prebid-creative-rendering-fix.md new file mode 100644 index 000000000..a21ec4d28 --- /dev/null +++ b/docs/superpowers/specs/2026-05-29-prebid-creative-rendering-fix.md @@ -0,0 +1,345 @@ +# Prebid Creative Rendering Fix Design + +_Author · 2026-05-29_ + +--- + +## 1. Problem Statement + +The Trusted Server server-side auction returns winning bids from PBS, but ads never +render on the Prebid path because `hb_adid` carries the wrong value. + +The Prebid Universal Creative in GAM constructs the creative fetch URL as: + +``` +https://?uuid= +``` + +TS currently sets `hb_adid` from `bid.adid` or `bid.id` (the OpenRTB bid ID / +impression ID). PBS actually caches the creative markup and returns the cache UUID +in `ext.prebid.cache.bids.cacheId`. The Universal Creative needs the **cache UUID**, +not the bid ID. The cache host and path are also not forwarded today. + +**Effect:** GAM receives a wrong UUID, fetches nothing, and the slot renders empty. + +--- + +## 2. Root Cause — Two Extraction Gaps + +### Gap 1: Wrong `hb_adid` source + +`prebid.rs` extracts: + +```rust +let ad_id = bid_obj + .get("adid") + .or_else(|| bid_obj.get("id")) // ← falls back to impression ID + .and_then(|v| v.as_str()) + .map(String::from); +``` + +Real PBS response has (in `ext.prebid.cache.bids`): + +```json +{ + "url": "https://openads.adsrvr.org/cache?uuid=f47447a0-b759-4f2f-9887-af458b79b570", + "cacheId": "f47447a0-b759-4f2f-9887-af458b79b570" +} +``` + +`bid.id` = `"ad-header-0-_R_4uapbsnql8alb_"` — the impression ID, useless to the +creative renderer. + +### Gap 2: Cache host and path not forwarded + +`build_bid_map` in `publisher.rs` emits `hb_pb`, `hb_bidder`, `hb_adid`, `nurl`, +`burl`. It does not emit `hb_cache_host` or `hb_cache_path`. The Prebid Universal +Creative needs both to construct the fetch URL. + +--- + +## 3. Non-Goals + +- APS creative rendering — APS does not use PBS Cache. APS creative delivery is + Amazon-owned and not addressed here. +- APS win detection over-fire — separate known limitation, separate issue. +- Dual bootstrap sync risk — separate maintenance issue. +- Slim-Prebid bundle — out of scope for Phase 1. + +--- + +## 4. Design + +### 4.1 New Fields on `Bid` (types.rs) + +Add three fields to `Bid` to carry the PBS Cache coordinates extracted from the bid +response: + +```rust +/// Prebid Cache UUID for this bid. Populated from +/// `ext.prebid.cache.bids.cacheId` in the PBS response. +/// Used as `hb_adid` targeting value in `window.tsjs.bids`. +/// None for non-PBS providers (e.g., APS) and PBS bids without cache enabled. +pub cache_id: Option, + +/// Prebid Cache host (e.g., `"openads.adsrvr.org"`). Populated from +/// the host component of `ext.prebid.cache.bids.url`. +/// Used as `hb_cache_host` targeting value. +pub cache_host: Option, + +/// Prebid Cache path (e.g., `"/cache"`). Populated from +/// the path component of `ext.prebid.cache.bids.url`. +/// Used as `hb_cache_path` targeting value. +pub cache_path: Option, +``` + +### 4.2 Extraction in `prebid.rs` + +In `parse_bid_object`, after extracting `nurl`/`burl`, extract the cache fields from +`ext.prebid.cache.bids`: + +```rust +// Extract PBS Cache coordinates from ext.prebid.cache.bids +let cache_entry = bid_obj + .get("ext") + .and_then(|e| e.get("prebid")) + .and_then(|p| p.get("cache")) + .and_then(|c| c.get("bids")); + +let cache_id = cache_entry + .and_then(|c| c.get("cacheId")) + .and_then(|v| v.as_str()) + .map(String::from); + +let (cache_host, cache_path) = cache_entry + .and_then(|c| c.get("url")) + .and_then(|v| v.as_str()) + .and_then(|url_str| { + url::Url::parse(url_str) + .map_err(|e| log::debug!("PBS cache URL parse failed: {}", e)) + .ok() + }) + .map(|u| { + let host = u.host_str().map(String::from); + // path() returns "/" for root — only use if non-trivial + let path = u.path().to_string(); + let path = if path.is_empty() || path == "/" { None } else { Some(path) }; + (host, path) + }) + .unwrap_or((None, None)); + +// Guard: if we extracted a cache UUID but couldn't extract the host, +// the bid will have hb_adid set but no endpoint to fetch from — creative will fail. +if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — \ + creative will fail to render for slot '{}'", + slot_id + ); +} +``` + +Note: `url` crate is already a workspace dependency. If not, parse host/path manually +by splitting on the first `/` after the scheme. + +The `ad_id` field (from `bid.adid` / `bid.id`) is **kept** — it maps to the OpenRTB +`adid` / `id` field that APS and other non-PBS providers may use. The cache fields are +**in addition**, not replacing `ad_id`. + +Populate all three fields on `AuctionBid`: + +```rust +Ok(AuctionBid { + ..., + ad_id, + cache_id, + cache_host, + cache_path, + ... +}) +``` + +### 4.3 `build_bid_map` in `publisher.rs` + +Priority for `hb_adid`: use `cache_id` when present (PBS path), fall back to `ad_id` +(APS / other providers, backward compat): + +```rust +// hb_adid: use PBS Cache UUID when present — the Prebid Universal Creative uses +// this as the cache lookup key, NOT the OpenRTB bid ID (bid.ad_id). Fall back to +// bid.ad_id for APS and other non-PBS providers. +let hb_adid = bid.cache_id.as_deref().or(bid.ad_id.as_deref()); +if let Some(id) = hb_adid { + obj.insert("hb_adid".to_string(), serde_json::Value::String(id.to_string())); +} + +// Cache coordinates — only present for PBS bids with Prebid Cache enabled +if let Some(ref host) = bid.cache_host { + obj.insert("hb_cache_host".to_string(), serde_json::Value::String(host.clone())); +} +if let Some(ref path) = bid.cache_path { + obj.insert("hb_cache_path".to_string(), serde_json::Value::String(path.clone())); +} +``` + +### 4.4 What `window.tsjs.bids` looks like after the fix + +```json +{ + "atf_sidebar_ad": { + "hb_pb": "0.01", + "hb_bidder": "thetradedesk", + "hb_adid": "f47447a0-b759-4f2f-9887-af458b79b570", + "hb_cache_host": "openads.adsrvr.org", + "hb_cache_path": "/cache", + "nurl": "https://...", + "burl": "https://..." + } +} +``` + +### 4.5 Win detection — no change required + +`slotRenderEnded` checks: + +```js +event.slot.getTargeting('hb_adid')[0] === bid.hb_adid +``` + +`adInit()` calls `setTargeting('hb_adid', cacheId)` with the cache UUID. +`event.slot.getTargeting('hb_adid')[0]` returns that same cache UUID. +`bid.hb_adid` is now also the cache UUID. +Match holds. No change to the win detection logic. + +### 4.6 GAM line item creative requirement (publisher action — not TS code) + +This is a **hard dependency outside the TS codebase**. The publisher must configure +GAM line items with a server-side compatible Prebid creative. The standard +client-side Universal Creative calls `pbjs.renderAd()` which requires Prebid.js to be +loaded — it will not be at first render (slim-Prebid loads post-`window.load`). + +The server-side compatible creative uses the `hb_cache_*` macros to fetch the markup +directly from PBS Cache: + +```html + +``` + +Alternatively, publishers using the Prebid Universal Creative package can use: + +```html + + +``` + +> **This creative configuration is a publisher/ad ops action, not a TS code change.** +> Document it in the integration guide and verify during onboarding. + +> **Cache TTL:** PBS Cache entries expire per the `bid.exp` field (default 300–3600s; +> the real response has `"exp": 3600`). Creative fetch must complete within this window. +> BFCache page restores after long idle sessions may hit expired cache entries — the +> creative will silently fail to render in that case. This is acceptable for Phase 1; +> the probability is low for typical session lengths. + +--- + +## 5. APS — Out of Scope + +APS does not use PBS Cache. APS bids will have `cache_id = None`, `cache_host = None`, +`cache_path = None`. The existing `ad_id` fallback path remains for APS. APS creative +rendering depends on Amazon's own GAM creative tag — separate from the Prebid path. + +APS win detection over-fires on the `!!bid.hb_bidder` fallback remain a known +limitation tracked separately. + +--- + +## 6. Files Changed + +| File | Change | +| ------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/auction/types.rs` | Add `cache_id`, `cache_host`, `cache_path` to `Bid` struct | +| `crates/trusted-server-core/src/integrations/prebid.rs` | Extract `ext.prebid.cache.bids.{cacheId,url}` in `parse_bid_object`; update `AuctionBid` → `Bid` conversion to carry the three new fields | +| `crates/trusted-server-core/src/publisher.rs` | `build_bid_map`: use `cache_id` for `hb_adid`, emit `hb_cache_host`/`hb_cache_path` | + +> **Implementer note — `AuctionBid` → `Bid` conversion:** `prebid.rs` constructs an +> intermediate `AuctionBid` type that is later converted to the shared `Bid` type from +> `types.rs`. The new `cache_id`, `cache_host`, `cache_path` fields must be added to +> **both** types and the conversion must map them explicitly. Verify by grepping for +> where `AuctionBid` is constructed and where it is converted to `Bid`; if they are the +> same type (a type alias), only one struct needs the new fields. If they differ, both +> need updating or the fields will silently be `None` in `build_bid_map`. + +Test files: +| File | Change | +|---|---| +| `crates/trusted-server-core/src/integrations/prebid.rs` tests | Add test: PBS response with cache entry → correct `hb_adid`, `hb_cache_host`, `hb_cache_path` injected | +| `crates/trusted-server-core/src/publisher.rs` tests | Add test: `build_bid_map` emits cache fields when present; falls back to `ad_id` when absent | + +--- + +## 7. Testing + +**Unit tests:** + +1. `prebid.rs`: bid with `ext.prebid.cache.bids.cacheId` → `bid.cache_id = Some(uuid)`, `bid.cache_host = Some("openads.adsrvr.org")`, `bid.cache_path = Some("/cache")` +2. `prebid.rs`: bid without `ext.prebid.cache` → `bid.cache_id = None`, `bid.cache_host = None`, `bid.cache_path = None` +3. `prebid.rs`: bid with only `adid` (no cache) → `bid.ad_id = Some(...)`, `bid.cache_id = None` +4. `prebid.rs`: bid with malformed cache URL → `cache_host = None`, `cache_path = None`, no panic +5. `publisher.rs` `build_bid_map`: bid with `cache_id` → `hb_adid` uses `cache_id`, `hb_cache_host`/`hb_cache_path` emitted +6. `publisher.rs` `build_bid_map`: bid with no `cache_id` but has `ad_id` → `hb_adid` falls back to `ad_id`, no cache keys emitted +7. `publisher.rs` `build_bid_map`: APS bid (no `cache_id`, no `ad_id`) → no `hb_adid` emitted +8. `types.rs`: `Bid` with all three cache fields round-trips through `serde_json::to_string` / `from_str` + +> **Note for implementer:** `make_bid()` or equivalent `Bid` construction helpers in test modules +> must be updated to initialise `cache_id`, `cache_host`, `cache_path` to `None` +> (they will fail to compile otherwise once the fields are added to the struct). + +**Integration verification (manual):** + +After deploying, verify `window.tsjs.bids` in browser devtools shows `hb_cache_host` +and `hb_cache_path` present. Verify `hb_adid` matches the UUID in +`ext.prebid.cache.bids.cacheId` from the raw PBS response. + +--- + +## 8. Rollout Dependency Checklist + +Before this fix has end-to-end effect: + +- [ ] TS: this PR merged and deployed +- [ ] GAM: publisher ad ops updates all Prebid line item creatives to the server-side + cache-fetch variant (see §4.6) +- [ ] PBS: Prebid Cache enabled and populated (confirmed from real response — already + working) +- [ ] Verify: `window.tsjs.bids` shows correct cache UUID in `hb_adid` after deploy + +--- + +## 9. Known Remaining Gaps (not in scope) + +| Gap | Severity | Tracking | +| ----------------------------------------------------------------- | -------- | ------------------ | +| APS win detection over-fires nurl/burl | P1 | Separate issue | +| Dual bootstrap (`gpt_bootstrap.js` + `installTsAdInit`) sync risk | P2 | Separate issue | +| Slim-Prebid bundle not yet built | Phase 2 | §9.8 of design doc | diff --git a/trusted-server.toml b/trusted-server.toml index 899c8c895..c7b7ec96e 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -125,6 +125,7 @@ enabled = false script_url = "https://securepubads.g.doubleclick.net/tag/js/gpt.js" cache_ttl_seconds = 3600 rewrite_script = true +# slim_prebid_url = "https://cdn.example.com/tsjs-prebid.min.js" # Consent forwarding configuration # Controls how Trusted Server interprets and forwards privacy consent signals. @@ -186,7 +187,7 @@ rewrite_script = true enabled = true providers = ["prebid", "aps"] mediator = "adserver_mock" -timeout_ms = 2000 +timeout_ms = 2000 # override per-publisher via TRUSTED_SERVER__AUCTION__TIMEOUT_MS # Context keys the JS client is allowed to forward into auction requests. # Keys not in this list are silently dropped. An empty list blocks all keys. allowed_context_keys = ["permutive_segments"] @@ -195,7 +196,7 @@ allowed_context_keys = ["permutive_segments"] enabled = true pub_id = "test-pub" endpoint = "https://origin-mocktioneer.cdintel.com/e/dtb/bid" -timeout_ms = 1000 +timeout_ms = 1000 # override per-publisher via TRUSTED_SERVER__INTEGRATIONS__APS__TIMEOUT_MS [integrations.google_tag_manager] enabled = false @@ -212,6 +213,10 @@ timeout_ms = 1000 # Inject before . # Visible in page source. Disable after investigation. # auction_html_comment = true +# +# Inject raw adm creative markup into window.tsjs.bids for GPT/GAM bridge +# debugging while PBS Cache is unavailable. NEVER enable in production. +# inject_adm_for_testing = true # Enable the JA4/TLS fingerprint debug endpoint at GET /_ts/debug/ja4. # Returns a plain-text response with the following fields (Fastly-observed values): # ja4 — JA4 TLS client fingerprint @@ -242,6 +247,53 @@ gam_network_id = "88059007" # drains in <50 ms but the auction runs to the limit. 500 ms is the recommended # default; raise only if your SSPs need more headroom and your analytics confirm # the DCL slip is acceptable. -auction_timeout_ms = 1500 +auction_timeout_ms = 1500 # override via TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__AUCTION_TIMEOUT_MS price_granularity = "dense" +# Slot templates — override entire array via: +# TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT='[{"id":"...","gam_unit_path":"...",...}]' + +[[creative_opportunities.slot]] +id = "atf_sidebar_ad" +gam_unit_path = "/a/b/news" +div_id = "div-ad-atf-sidebar" +page_patterns = ["/20**", "/news/**"] +formats = [{ width = 300, height = 250 }] +floor_price = 0.50 + +[creative_opportunities.slot.targeting] +pos = "atf" +zone = "atfSidebar" + +[creative_opportunities.slot.providers.aps] +slot_id = "aps-slot-atf-sidebar" + +[[creative_opportunities.slot]] +id = "homepage_header_ad" +gam_unit_path = "/a/b/homepage" +div_id = "div-ad-homepage-header" +page_patterns = ["/"] +formats = [{ width = 728, height = 90 }] +floor_price = 0.50 + +[creative_opportunities.slot.targeting] +pos = "atf" +zone = "header" + +[creative_opportunities.slot.providers.aps] +slot_id = "aps-slot-homepage-header" + +[[creative_opportunities.slot]] +id = "homepage_footer_ad" +gam_unit_path = "/a/b/homepage" +div_id = "div-ad-homepage-footer" +page_patterns = ["/"] +formats = [{ width = 728, height = 90 }, { width = 768, height = 66 }] +floor_price = 0.50 + +[creative_opportunities.slot.targeting] +pos = "btf" +zone = "fixedBottom" + +[creative_opportunities.slot.providers.aps] +slot_id = "aps-slot-homepage-footer" From b77ebc4d1a50110066d805cb0d760d339cb62b7a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 9 Jun 2026 16:55:00 +0530 Subject: [PATCH 074/395] Wire KV-enriched EID resolution into server-side auction paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both handle_publisher_request and handle_page_bids now run the full four-step EID pipeline (resolve_client_auction_eids → resolve_auction_eids → merge_auction_eids → gate_eids_by_consent) matching the client-side /auction endpoint. Previously both paths called parse_ts_eids_cookie, which read only the ts-eids browser cookie and skipped the KV identity graph lookup entirely. AuctionDispatch gains a registry field so the partner registry reaches handle_publisher_request without exceeding the seven-argument limit. handle_page_bids gains kv and registry parameters for the same reason. parse_ts_eids_cookie is moved to #[cfg(test)] as it is now test-only. --- .../trusted-server-adapter-fastly/src/main.rs | 49 +++++++------ .../src/auction/endpoints.rs | 6 +- crates/trusted-server-core/src/cookies.rs | 10 +-- .../src/creative_opportunities.rs | 8 ++- crates/trusted-server-core/src/publisher.rs | 69 ++++++++++++++----- 5 files changed, 97 insertions(+), 45 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 6f95373a1..7d8b98e48 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -328,6 +328,12 @@ async fn route_request( let path = req.get_path().to_string(); let method = req.get_method().clone(); + let registry_ref = if partner_registry.is_empty() { + None + } else { + Some(partner_registry) + }; + // Match known routes and handle them let (result, organic_route) = match (method, path.as_str()) { // Serve the tsjs library @@ -368,30 +374,32 @@ async fn route_request( } // Unified auction endpoint (returns creative HTML inline) - (Method::POST, "/auction") => { - let registry_ref = if partner_registry.is_empty() { - None - } else { - Some(partner_registry) - }; - ( - handle_auction( - settings, - orchestrator, - kv_graph.as_ref(), - registry_ref, - &ec_context, - runtime_services, - req, - ) - .await, - false, + (Method::POST, "/auction") => ( + handle_auction( + settings, + orchestrator, + kv_graph.as_ref(), + registry_ref, + &ec_context, + runtime_services, + req, ) - } + .await, + false, + ), // SPA/CSR navigation endpoint — returns slots + bids JSON for the given path (Method::GET, "/__ts/page-bids") => ( - handle_page_bids(settings, orchestrator, runtime_services, slots, req).await, + handle_page_bids( + settings, + orchestrator, + runtime_services, + kv_graph.as_ref(), + registry_ref, + slots, + req, + ) + .await, false, ), @@ -443,6 +451,7 @@ async fn route_request( trusted_server_core::publisher::AuctionDispatch { orchestrator, slots, + registry: registry_ref, }, req, ) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 84d8b3f3b..f1c010de1 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -208,7 +208,7 @@ pub async fn handle_auction( /// Returns `None` when any prerequisite is missing (no KV store, no partner /// store, no EC, consent denied). On KV or partner-resolution errors, logs a /// warning and returns empty EIDs so the auction can proceed in degraded mode. -fn resolve_auction_eids( +pub(crate) fn resolve_auction_eids( kv: Option<&KvIdentityGraph>, registry: Option<&PartnerRegistry>, ec_context: &EcContext, @@ -251,7 +251,7 @@ fn extract_cookie_value(req: &Request, name: &str) -> Option { None } -fn resolve_client_auction_eids( +pub(crate) fn resolve_client_auction_eids( raw: Option<&JsonValue>, cookie_value: Option<&str>, ) -> Option> { @@ -347,7 +347,7 @@ fn parse_client_auction_uid(raw: &JsonValue) -> Option { Some(Uid { id, atype, ext }) } -fn merge_auction_eids( +pub(crate) fn merge_auction_eids( client_eids: Option>, resolved_eids: Option>, ) -> Option> { diff --git a/crates/trusted-server-core/src/cookies.rs b/crates/trusted-server-core/src/cookies.rs index 302e35cea..2d558e314 100644 --- a/crates/trusted-server-core/src/cookies.rs +++ b/crates/trusted-server-core/src/cookies.rs @@ -3,17 +3,18 @@ //! This module provides functionality for parsing, stripping, and forwarding cookies //! used in the trusted server system. -use base64::{engine::general_purpose::STANDARD, Engine as _}; use cookie::{Cookie, CookieJar}; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::header; use http::Request; -use crate::constants::{ - COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_TS_EIDS, COOKIE_US_PRIVACY, -}; +#[cfg(test)] +use crate::constants::COOKIE_TS_EIDS; +use crate::constants::{COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_US_PRIVACY}; use crate::error::TrustedServerError; +#[cfg(test)] +use base64::{engine::general_purpose::STANDARD, Engine as _}; /// Cookie names carrying privacy consent signals. /// @@ -81,6 +82,7 @@ pub fn handle_request_cookies( /// Returns `None` if the cookie is absent, base64-malformed, JSON-malformed, /// or the decoded array is empty. Parse failures are logged at `debug` level /// so operators can diagnose JS SDK / server mismatches. +#[cfg(test)] #[must_use] pub(crate) fn parse_ts_eids_cookie(jar: Option<&CookieJar>) -> Option> { let value = jar?.get(COOKIE_TS_EIDS)?.value().to_owned(); diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 2df61bb0c..a7b3d579a 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -457,7 +457,8 @@ mod tests { #[test] fn to_ad_slot_injects_trusted_server_when_prebid_bidders_empty() { let mut slot = make_slot("header", vec!["/"]); - slot.targeting.insert("zone".to_string(), "header".to_string()); + slot.targeting + .insert("zone".to_string(), "header".to_string()); slot.providers.prebid = Some(PrebidSlotParams { bidders: HashMap::new(), }); @@ -515,7 +516,10 @@ mod tests { .bidders .get("mocktioneer") .expect("should have mocktioneer bidder"); - assert_eq!(params.get("custom").and_then(serde_json::Value::as_bool), Some(true)); + assert_eq!( + params.get("custom").and_then(serde_json::Value::as_bool), + Some(true) + ); } #[test] diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1bcd614c7..823977df5 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -18,15 +18,20 @@ use error_stack::{Report, ResultExt}; use fastly::http::{header, StatusCode}; use fastly::{Body, Request, Response}; +use crate::auction::endpoints::{ + merge_auction_eids, resolve_auction_eids, resolve_client_auction_eids, +}; use crate::auction::orchestrator::{AuctionOrchestrator, DispatchedAuction}; use crate::auction::types::{ AuctionContext, AuctionRequest, Bid, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, }; use crate::backend::BackendConfig; use crate::compat; -use crate::constants::HEADER_X_COMPRESS_HINT; -use crate::cookies::{handle_request_cookies, parse_ts_eids_cookie}; +use crate::consent::gate_eids_by_consent; +use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; +use crate::cookies::handle_request_cookies; use crate::ec::kv::KvIdentityGraph; +use crate::ec::registry::PartnerRegistry; use crate::ec::EcContext; use crate::error::TrustedServerError; use crate::http_util::{is_navigation_request, serve_static_with_etag, RequestInfo}; @@ -810,6 +815,8 @@ pub struct AuctionDispatch<'a> { pub orchestrator: &'a crate::auction::orchestrator::AuctionOrchestrator, /// Creative opportunity slot definitions matched against the request path. pub slots: &'a [crate::creative_opportunities::CreativeOpportunitySlot], + /// Partner registry for KV-backed EID resolution. `None` skips KV enrichment. + pub registry: Option<&'a PartnerRegistry>, } /// Proxies requests to the publisher's origin server. @@ -968,7 +975,19 @@ pub async fn handle_publisher_request( &request_info, req.get_header_str("user-agent"), ); - auction_request.user.eids = parse_ts_eids_cookie(cookie_jar.as_ref()); + let ts_eids_value = cookie_jar + .as_ref() + .and_then(|j| j.get(COOKIE_TS_EIDS)) + .map(|c| c.value().to_owned()); + let client_eids = resolve_client_auction_eids(None, ts_eids_value.as_deref()); + let kv_eids = resolve_auction_eids(kv, auction.registry, ec_context); + let merged_eids = merge_auction_eids(client_eids, kv_eids); + let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); + auction_request.user.eids = + gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); + if had_eids && auction_request.user.eids.is_none() { + log::warn!("Server-side auction EIDs stripped by TCF consent gating"); + } let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); if client_ip.is_some() || geo.is_some() { let device = auction_request.device.get_or_insert(DeviceInfo { @@ -1456,6 +1475,8 @@ pub async fn handle_page_bids( settings: &Settings, orchestrator: &AuctionOrchestrator, services: &RuntimeServices, + kv: Option<&KvIdentityGraph>, + registry: Option<&PartnerRegistry>, slots: &[crate::creative_opportunities::CreativeOpportunitySlot], req: Request, ) -> Result> { @@ -1527,7 +1548,19 @@ pub async fn handle_page_bids( &request_info, req.get_header_str("user-agent"), ); - auction_request.user.eids = parse_ts_eids_cookie(cookie_jar.as_ref()); + let ts_eids_value = cookie_jar + .as_ref() + .and_then(|j| j.get(COOKIE_TS_EIDS)) + .map(|c| c.value().to_owned()); + let client_eids = resolve_client_auction_eids(None, ts_eids_value.as_deref()); + let kv_eids = resolve_auction_eids(kv, registry, &ec_ctx); + let merged_eids = merge_auction_eids(client_eids, kv_eids); + let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); + auction_request.user.eids = + gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); + if had_eids && auction_request.user.eids.is_none() { + log::warn!("Page-bids auction EIDs stripped by TCF consent gating"); + } let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); if client_ip.is_some() || geo.is_some() { let device = auction_request.device.get_or_insert(DeviceInfo { @@ -3133,9 +3166,10 @@ mod tests { let services = noop_services(); let req = make_page_bids_request("/2024/01/my-article/"); - let response = handle_page_bids(&settings, &orchestrator, &services, &[], req) - .await - .expect("should return ok response"); + let response = + handle_page_bids(&settings, &orchestrator, &services, None, None, &[], req) + .await + .expect("should return ok response"); let body: serde_json::Value = serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); @@ -3170,9 +3204,10 @@ mod tests { let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1)"); - let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) - .await - .expect("should return ok response"); + let response = + handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) + .await + .expect("should return ok response"); let body: serde_json::Value = serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); @@ -3206,9 +3241,10 @@ mod tests { let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("sec-purpose", "prefetch"); - let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) - .await - .expect("should return ok response"); + let response = + handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) + .await + .expect("should return ok response"); let body: serde_json::Value = serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); @@ -3240,9 +3276,10 @@ mod tests { let slots = article_slot(); // slot matches /20** only let req = make_page_bids_request("/about"); // does not match - let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) - .await - .expect("should return ok response"); + let response = + handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) + .await + .expect("should return ok response"); let body: serde_json::Value = serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); From 321fbafb3dc1123bba292ca5a125423d1c14b77c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 9 Jun 2026 16:58:24 +0530 Subject: [PATCH 075/395] Remove dead build.rs from trusted-server-adapter-fastly The file only emitted a rerun-if-changed watch for creative-opportunities.toml, which was deleted when slot config was consolidated into trusted-server.toml. Config validation now runs entirely in trusted-server-core/build.rs. --- crates/trusted-server-adapter-fastly/build.rs | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 crates/trusted-server-adapter-fastly/build.rs diff --git a/crates/trusted-server-adapter-fastly/build.rs b/crates/trusted-server-adapter-fastly/build.rs deleted file mode 100644 index 0ad1f2dd9..000000000 --- a/crates/trusted-server-adapter-fastly/build.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("cargo:rerun-if-changed=../../../creative-opportunities.toml"); -} From 459fe60179a89ca8057929f77a44d3cef17755b8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 9 Jun 2026 17:04:19 +0530 Subject: [PATCH 076/395] Fix CI failures: update integration-tests lock file and prefer-const lint error --- crates/integration-tests/Cargo.lock | 237 +++++++++++--------- crates/js/lib/src/integrations/gpt/index.ts | 2 +- 2 files changed, 127 insertions(+), 112 deletions(-) diff --git a/crates/integration-tests/Cargo.lock b/crates/integration-tests/Cargo.lock index 9f80a0ef7..40fbe0039 100644 --- a/crates/integration-tests/Cargo.lock +++ b/crates/integration-tests/Cargo.lock @@ -201,9 +201,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "axum" @@ -274,9 +274,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" dependencies = [ "serde_core", ] @@ -316,7 +316,7 @@ checksum = "87a52479c9237eb04047ddb94788c41ca0d26eaff8b697ecfbb4c32f7fdc3b1b" dependencies = [ "async-stream", "base64", - "bitflags 2.11.1", + "bitflags 2.13.0", "bollard-buildkit-proto", "bollard-stubs", "bytes", @@ -387,9 +387,9 @@ dependencies = [ [[package]] name = "brotli" -version = "8.0.2" +version = "8.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -398,9 +398,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.0" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -423,9 +423,9 @@ checksum = "d8e6738dfb11354886f890621b4a34c0b177f75538023f7100b608ab9adbd66b" [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "byteorder" @@ -441,9 +441,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.62" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", "shlex", @@ -481,9 +481,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -530,9 +530,9 @@ checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "config" -version = "0.15.22" +version = "0.15.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68cfe19cd7d23ffde002c24ffa5cda73931913ef394d5eaaa32037dc940c0c" +checksum = "f316c6237b2d38be61949ecd15268a4c6ca32570079394a2444d9ce2c72a72d8" dependencies = [ "async-trait", "convert_case 0.6.0", @@ -903,9 +903,9 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -923,9 +923,9 @@ dependencies = [ [[package]] name = "docker_credential" -version = "1.3.3" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4564c274ebf369f501de192b02a0b81a5c4bda375abfe526aa70fc702fa6fa0" +checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" dependencies = [ "base64", "serde", @@ -1043,9 +1043,9 @@ checksum = "7c6ba7d4eec39eaa9ab24d44a0e73a7949a1095a8b3f3abb11eddf27dbb56a53" [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "elliptic-curve" @@ -1469,6 +1469,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "group" version = "0.13.0" @@ -1520,6 +1526,15 @@ dependencies = [ "foldhash 0.1.5", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -1533,11 +1548,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -1584,9 +1599,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1629,9 +1644,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -2005,9 +2020,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.24" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" dependencies = [ "jiff-static", "log", @@ -2018,9 +2033,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.24" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", @@ -2065,13 +2080,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.98" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -2142,9 +2156,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "lol_html" @@ -2152,7 +2166,7 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00aad58f6ec3990e795943872f13651e7a5fa59dca2c8f31a74faf8a0e0fb652" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "cssparser 0.36.0", "encoding_rs", @@ -2210,9 +2224,9 @@ checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "mime" @@ -2232,9 +2246,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -2324,9 +2338,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-derive" @@ -2400,11 +2414,11 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.79" +version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "foreign-types", "libc", @@ -2431,9 +2445,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.115" +version = "0.9.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", @@ -2840,9 +2854,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -2850,9 +2864,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools 0.14.0", @@ -2863,9 +2877,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ "prost", ] @@ -2972,7 +2986,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", ] [[package]] @@ -3088,7 +3102,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4147b952f3f819eca0e99527022f7d6a8d05f111aeb0a62960c74eb283bec8fc" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "once_cell", "serde", "serde_derive", @@ -3147,7 +3161,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys", @@ -3171,9 +3185,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -3305,7 +3319,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -3328,7 +3342,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cssparser 0.34.0", "derive_more 0.99.20", "fxhash", @@ -3347,7 +3361,7 @@ version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cfaaa6035167f0e604e42723c7650d59ee269ef220d7bbe0565602c8a0173b9" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cssparser 0.36.0", "derive_more 2.1.1", "log", @@ -3410,9 +3424,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -3455,9 +3469,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64", "bs58", @@ -3475,9 +3489,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -3520,9 +3534,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signature" @@ -3560,9 +3574,9 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -3710,7 +3724,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4053,11 +4067,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "bytes", "futures-util", "http", @@ -4131,6 +4145,7 @@ dependencies = [ "fastly", "flate2", "futures", + "glob", "hex", "hmac", "http", @@ -4189,9 +4204,9 @@ checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" @@ -4217,9 +4232,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" @@ -4321,9 +4336,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.1" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -4417,9 +4432,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" dependencies = [ "cfg-if", "once_cell", @@ -4430,9 +4445,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.71" +version = "0.4.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" dependencies = [ "js-sys", "wasm-bindgen", @@ -4440,9 +4455,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4450,9 +4465,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" dependencies = [ "bumpalo", "proc-macro2", @@ -4463,9 +4478,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" dependencies = [ "unicode-ident", ] @@ -4498,7 +4513,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "hashbrown 0.15.5", "indexmap 2.14.0", "semver", @@ -4506,9 +4521,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.98" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" dependencies = [ "js-sys", "wasm-bindgen", @@ -4526,9 +4541,9 @@ dependencies = [ [[package]] name = "which" -version = "8.0.2" +version = "8.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81995fafaaaf6ae47a7d0cc83c67caf92aeb7e5331650ae6ff856f7c0c60c459" +checksum = "c789537cf2f7f55be8e6192f92e464174ee55f91af622777f7f1ceb0dbccd03e" dependencies = [ "libc", ] @@ -4740,7 +4755,7 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", ] [[package]] @@ -4758,7 +4773,7 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", ] [[package]] @@ -4810,7 +4825,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.1", + "bitflags 2.13.0", "indexmap 2.14.0", "log", "serde", @@ -4858,9 +4873,9 @@ dependencies = [ [[package]] name = "yaml-rust2" -version = "0.10.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2462ea039c445496d8793d052e13787f2b90e750b833afee748e601c17621ed9" +checksum = "631a50d867fafb7093e709d75aaee9e0e0d5deb934021fcea25ac2fe09edc51e" dependencies = [ "arraydeque", "encoding_rs", @@ -4869,9 +4884,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4892,18 +4907,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 4276bc7b8..1d11571bd 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -203,7 +203,7 @@ function injectAdmIntoSlot(divId: string, adm: string): void { try { // divId may be the container div (used by GPT slot) or the inner div. // Search both so we can find the GAM iframe wherever it was rendered. - let slotEl = document.getElementById(divId); + const slotEl = document.getElementById(divId); if (!slotEl) return; // Extract the first iframe src from the adm (e.g. mocktioneer creative From 66140220c932efe55cc15da4c1f88fef42dc6626 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 9 Jun 2026 17:09:01 +0530 Subject: [PATCH 077/395] Update workspace Cargo.lock to resolve shared dependency version mismatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligns log (0.4.29 → 0.4.32) and serde_json (1.0.149 → 1.0.150) with the versions already pulled into crates/integration-tests/Cargo.lock. --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5fd679f6a..2d1ad743c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1582,9 +1582,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "log-fastly" @@ -2270,9 +2270,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", From e6f5fc890ff7875b3c8c19ed88c440b95b48b38b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 9 Jun 2026 17:33:59 +0530 Subject: [PATCH 078/395] Update spec to reflect consolidated slot config and current global namespace Replace all references to the deleted `creative-opportunities.toml` file with the `[creative_opportunities]` section in `trusted-server.toml`. Update all `window.__ts_*` global name references to the current `window.tsjs.*` namespace (tsjs.bids, tsjs.adSlots, tsjs.adInit). --- ...6-04-15-server-side-ad-templates-design.md | 144 +++++++++--------- 1 file changed, 73 insertions(+), 71 deletions(-) diff --git a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md index 94fe1999a..bdf24ff9c 100644 --- a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md +++ b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md @@ -89,13 +89,14 @@ across every navigation in the user's clickstream rather than once per session. ## 4. Architecture -### 4.1 New File: `creative-opportunities.toml` +### 4.1 Slot configuration in `trusted-server.toml` -A new config file at the repo root, alongside `trusted-server.toml`. It holds all slot -templates: page pattern matching rules, ad formats, floor prices, GAM targeting -key-values, and per-provider bidder params. PBS bidder-level params (placement IDs, -account IDs) live in Prebid Server stored requests, keyed by slot ID. APS params are -specified inline per slot under `[slot.providers.aps]`. +Slot templates live in `trusted-server.toml` under `[[creative_opportunities.slot]]` +(consolidated from the original `creative-opportunities.toml`). Each entry holds page +pattern matching rules, ad formats, floor prices, GAM targeting key-values, and +per-provider bidder params. PBS bidder-level params (placement IDs, account IDs) live +in Prebid Server stored requests, keyed by slot ID. APS params are specified inline per +slot under `[slot.providers.aps]`. Loaded at build time via `include_str!()` and compiled into the WASM binary. Slot changes require a redeploy; this is intentional (fast reads, no KV overhead, no @@ -103,7 +104,7 @@ per-request cost). A migration path to KV-backed config is tracked in §9.5. `floor_price` is the publisher-owned hard floor per slot — the source of truth for the minimum acceptable bid price, enforced at the edge before bids reach the ad server. Any -bid below the floor is discarded at the orchestrator level before it enters `__ts_bids`. +bid below the floor is discarded at the orchestrator level before it enters `tsjs.bids`. SSPs may apply their own dynamic floors independently within their platforms; this floor is the publisher's baseline that supersedes all other floor logic by virtue of being enforced earliest in the pipeline. @@ -118,7 +119,7 @@ gam_network_id = "21765378893" # Optional. Defaults to [auction].timeout_ms if not set. # Recommended: 500ms (vs client-side 1000–1500ms) due to lower edge→PBS RTT. # This value is also the upper bound on the -close hold; once A_deadline -# fires, TS injects an empty __ts_bids and emits regardless. +# fires, TS injects an empty tsjs.bids and emits regardless. auction_timeout_ms = 500 # Granularity table for hb_pb price bucket strings. @@ -127,7 +128,7 @@ auction_timeout_ms = 500 price_granularity = "dense" ``` -#### `creative-opportunities.toml` schema +#### `[creative_opportunities]` schema ```toml [[slot]] @@ -278,10 +279,10 @@ request. Before firing, TS gates on: skip the auction. Avoids spending auction inventory on speculative navigations that may never paint. - **Method** — only `GET` requests trigger auctions. `HEAD` requests skip. -- **Slot match** — at least one slot in `creative-opportunities.toml` must match the +- **Slot match** — at least one slot in `[creative_opportunities]` (in `trusted-server.toml`) must match the request path. Empty match = no auction. -Skipped auctions emit no `__ts_bids` and let the page proceed unmodified by the ad +Skipped auctions emit no `tsjs.bids` and let the page proceed unmodified by the ad stack. Skipped requests still benefit from the EC cookie set / KV identity update paths that run independently of the auction. @@ -294,7 +295,7 @@ existing EC pipeline and is the load-bearing identity input to the auction (see Consent gating: - If consent is **absent or denied** (no TCF consent string, or purpose 1 not consented): - the auction is not fired. `__ts_bids` is omitted from the page. GPT falls back to its + the auction is not fired. `tsjs.bids` is omitted from the page. GPT falls back to its own auction. This is treated as a first-class edge case in §8. - **Mid-page consent revocation** is out of scope for Phase 1; bids already injected remain. Phase 2 will address consent event propagation. @@ -313,8 +314,7 @@ The orchestrator's existing behavior is unchanged: (`creative_opportunities.auction_timeout_ms`, falling back to `[auction].timeout_ms`) - Floor price filtering, bid unification, and winning bid selection are applied as today - PBS resolves bidder params from its stored requests by slot ID -- APS bidder params are read from `[slot.providers.aps]` in - `creative-opportunities.toml` +- APS bidder params are read from `[slot.providers.aps]` in `trusted-server.toml` #### The bounded `` hold @@ -344,7 +344,7 @@ In English: finished by the time we need it because we waited for origin too. - If origin drains before the auction completes: body close held until either auction completes or `A_deadline` fires. Hold is bounded by `A_deadline`. -- If `A_deadline` fires first: TS injects `__ts_bids = {}` (graceful no-bid fallback) +- If `A_deadline` fires first: TS injects `tsjs.bids = {}` (graceful no-bid fallback) and emits the close tag. GPT proceeds without bid targeting; GAM runs its own auction. This is the **soft inner deadline watchdog** — auction overrun never blocks the page past `A_deadline`. @@ -371,12 +371,12 @@ and resource load time, exactly the same as a page without TS in the path. TS injects two `, ContentType::Html)`. @@ -446,7 +446,7 @@ task, fallback to `{}` on watchdog) and calls > U+2029 are unicode-escaped to neutralize any markup that could break out of the > `", + b"", + ], + Arc::clone(&read_count), + ); + let mut processor = RecordingProcessor { + read_count: Arc::clone(&read_count), + body_close_processed_at: Arc::clone(&body_close_processed_at), + }; + let ad_bids_state = Arc::new(Mutex::new(None)); + let ctx = AuctionCollectCtx { + dispatched, + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + }; + let mut output = Vec::new(); + + body_close_hold_loop(reader, &mut output, &mut processor, ctx) + .await + .expect("should stream body with auction hold"); + + assert_eq!( + body_close_processed_at.load(Ordering::SeqCst), + 1, + "close-body tail should be processed as soon as it is found, before later chunks are read" + ); + assert_eq!( + std::str::from_utf8(&output).expect("should be utf8"), + "painted", + "post-body chunks should still stream in order" + ); + } + #[test] fn body_close_hold_buffer_holds_close_body_tail_in_single_chunk() { let mut hold = BodyCloseHoldBuffer::new(); diff --git a/trusted-server.toml b/trusted-server.toml index 73f225a18..e1c35e11d 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -215,7 +215,7 @@ rewrite_script = true [auction] enabled = true providers = ["prebid", "aps"] -mediator = "adserver_mock" +# mediator = "adserver_mock" timeout_ms = 2000 # override per-publisher via TRUSTED_SERVER__AUCTION__TIMEOUT_MS # Context keys the JS client is allowed to forward into auction requests. # Keys not in this list are silently dropped. An empty list blocks all keys. From def951ab8f2b3c560a1694a570ca48b133738ab7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 10 Jun 2026 16:25:56 +0530 Subject: [PATCH 083/395] Add per-bidder Prebid nurl suppression and refresh metadata --- .env.example | 1 + .../js/lib/src/integrations/prebid/index.ts | 118 +++++++++++++++--- .../test/integrations/prebid/index.test.ts | 70 +++++++++++ .../src/integrations/prebid.rs | 66 +++++++++- docs/guide/configuration.md | 2 + docs/guide/integrations/prebid.md | 2 + ...6-04-15-server-side-ad-templates-design.md | 11 +- trusted-server.toml | 2 + 8 files changed, 247 insertions(+), 25 deletions(-) diff --git a/.env.example b/.env.example index 1121ecd9b..cec5d91de 100644 --- a/.env.example +++ b/.env.example @@ -45,6 +45,7 @@ TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=false # TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_ZONE_OVERRIDES='{"kargo":{"header":{"placementId":"_abc"}}}' # Preferred canonical env shape for future generic rules # TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDE_RULES='[{"when":{"bidder":"kargo","zone":"header"},"set":{"placementId":"_abc"}}]' +# TRUSTED_SERVER__INTEGRATIONS__PREBID__SUPPRESS_NURL_BIDDERS=exampleBidder,anotherBidder # TRUSTED_SERVER__INTEGRATIONS__PREBID__AUTO_CONFIGURE=false # TRUSTED_SERVER__INTEGRATIONS__PREBID__DEBUG=false # TRUSTED_SERVER__INTEGRATIONS__PREBID__TEST_MODE=false diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index c395905ef..feb31e1a9 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -32,6 +32,7 @@ import './_user_ids.generated'; import { log } from '../../core/log'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import type { AuctionBid, AuctionEid } from '../../core/auction'; +import type { AuctionSlot } from '../../core/types'; import { DEFAULT_PREBID_USER_ID_MODULES, PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; @@ -212,7 +213,13 @@ export function auctionBidsToPrebidBids(auctionBids: AuctionBid[], bidRequests: type PbjsConfig = Parameters[0]; type TrustedServerBid = { bidder?: string; params?: Record }; -type TrustedServerAdUnit = { code?: string; bids?: TrustedServerBid[] }; +type BannerSize = [number, number]; +type TrustedServerBanner = { sizes: BannerSize[]; name?: string }; +type TrustedServerAdUnit = { + code?: string; + mediaTypes?: { banner?: TrustedServerBanner }; + bids?: TrustedServerBid[]; +}; type TrustedServerBidRequest = { adUnitCode?: string; code?: string; @@ -232,6 +239,17 @@ type PrebidUserIdEid = { uids?: Array<{ id?: unknown; atype?: unknown; ext?: unknown }>; }; +type RefreshGptSlot = { + getSlotElementId?: () => string; + getTargeting?: (key: string) => string[]; + getSizes?: () => unknown[]; +}; + +const DEFAULT_REFRESH_SIZES: BannerSize[] = [ + [728, 90], + [300, 250], +]; + function sanitizeAuctionUid(uid: { id?: unknown; atype?: unknown; @@ -258,6 +276,63 @@ function isDefined(value: T | undefined): value is T { return value !== undefined; } +function isPositiveFiniteNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) && value > 0; +} + +function parseBannerSize(size: unknown): BannerSize | undefined { + if (Array.isArray(size) && isPositiveFiniteNumber(size[0]) && isPositiveFiniteNumber(size[1])) { + return [size[0], size[1]]; + } + + const gptSize = size as { getWidth?: () => unknown; getHeight?: () => unknown }; + const width = gptSize?.getWidth?.(); + const height = gptSize?.getHeight?.(); + if (isPositiveFiniteNumber(width) && isPositiveFiniteNumber(height)) { + return [width, height]; + } + + return undefined; +} + +function bannerSizesFromGptSlot(slot: RefreshGptSlot): BannerSize[] | undefined { + const sizes = slot.getSizes?.(); + if (!Array.isArray(sizes)) { + return undefined; + } + + const parsedSizes = sizes.map(parseBannerSize).filter(isDefined); + return parsedSizes.length > 0 ? parsedSizes : undefined; +} + +function bannerSizesFromInjectedSlot(slot: AuctionSlot | undefined): BannerSize[] | undefined { + const parsedSizes = slot?.formats?.map(parseBannerSize).filter(isDefined) ?? []; + return parsedSizes.length > 0 ? parsedSizes : undefined; +} + +function refreshSlotElementId(slot: RefreshGptSlot): string | undefined { + const elementId = slot.getSlotElementId?.(); + return elementId && elementId.length > 0 ? elementId : undefined; +} + +function findInjectedSlotForRefresh(slot: RefreshGptSlot): AuctionSlot | undefined { + const elementId = refreshSlotElementId(slot); + if (!elementId) { + return undefined; + } + + return window.tsjs?.adSlots?.find( + (adSlot) => + elementId === adSlot.div_id || + elementId === `${adSlot.div_id}-container` || + elementId.startsWith(adSlot.div_id) + ); +} + +function firstTargetingValue(values: string[] | undefined): string | undefined { + return values?.find((value) => value.length > 0); +} + function collectAuctionEids(): AuctionEid[] | undefined { if (typeof pbjs.getUserIdsAsEids !== 'function') { return undefined; @@ -524,13 +599,15 @@ export function installRefreshHandler(timeoutMs = 1500): void { pubads.refresh = function (slots?: unknown[], opts?: unknown) { // For bare refresh() calls (no slots arg), get all registered slots from GPT // so we can filter out TS first-impression slots and auction the rest. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const targetSlots: any[] = slots ?? (pubads as any).getSlots?.() ?? []; + const targetSlots = ( + slots ?? + (pubads as { getSlots?: () => unknown[] }).getSlots?.() ?? + [] + ).filter((slot): slot is RefreshGptSlot => typeof slot === 'object' && slot !== null); // Filter out TS first-impression slots — they don't need client-side refresh auctions. const nonTsSlots = targetSlots.filter( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (s: any) => !s.getTargeting?.('ts_initial')?.includes('1') + (slot) => !slot.getTargeting?.('ts_initial')?.includes('1') ); if (!nonTsSlots.length) { @@ -538,19 +615,24 @@ export function installRefreshHandler(timeoutMs = 1500): void { return originalRefresh(slots, opts); } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const adUnits = nonTsSlots.map((s: any) => ({ - code: s.getSlotElementId?.() ?? s, - mediaTypes: { - banner: { - sizes: [ - [728, 90], - [300, 250], - ] as [number, number][], - }, - }, - bids: [{ bidder: ADAPTER_CODE, params: { zone: 'refresh' } }], - })); + const adUnits = nonTsSlots.map((slot) => { + const injectedSlot = findInjectedSlotForRefresh(slot); + const zone = + injectedSlot?.targeting?.[ZONE_KEY] ?? firstTargetingValue(slot.getTargeting?.(ZONE_KEY)); + const banner: TrustedServerBanner = { + sizes: + bannerSizesFromInjectedSlot(injectedSlot) ?? + bannerSizesFromGptSlot(slot) ?? + DEFAULT_REFRESH_SIZES, + ...(zone ? { name: zone } : {}), + }; + + return { + code: refreshSlotElementId(slot) ?? 'refresh-slot', + mediaTypes: { banner }, + bids: [{ bidder: ADAPTER_CODE, params: zone ? { [ZONE_KEY]: zone } : {} }], + }; + }); pbjs.requestBids({ adUnits, diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index f12345d25..c79bfd080 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -62,6 +62,7 @@ import { getInjectedConfig, auctionBidsToPrebidBids, installPrebidNpm, + installRefreshHandler, } from '../../../src/integrations/prebid/index'; import type { AuctionBid } from '../../../src/core/auction'; @@ -765,6 +766,75 @@ describe('prebid/installPrebidNpm with server-injected config', () => { }); }); +describe('prebid/installRefreshHandler', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockPbjs.requestBids = mockRequestBids; + mockPbjs.adUnits = []; + (window as any).tsjs = undefined; + delete (window as any).googletag; + }); + + afterEach(() => { + (window as any).tsjs = undefined; + delete (window as any).googletag; + }); + + it('builds refresh ad units from injected slot metadata', () => { + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [ + [970, 250], + [728, 90], + ], + targeting: { zone: 'homepage', pos: 'atf' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + timeout: 750, + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + mediaTypes: { + banner: { + name: 'homepage', + sizes: [ + [970, 250], + [728, 90], + ], + }, + }, + bids: [{ bidder: 'trustedServer', params: { zone: 'homepage' } }], + }), + ], + }) + ); + }); +}); + describe('prebid/client-side bidders', () => { beforeEach(() => { vi.clearAllMocks(); diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 6c0ee0eaa..b757a1bf6 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -189,6 +189,14 @@ pub struct PrebidIntegrationConfig { /// client does not double-fire them via `sendBeacon`. Default: `false`. #[serde(default)] pub suppress_nurl: bool, + /// Bidder seats whose `nurl` and `burl` should be stripped before they reach + /// `window.tsjs.bids`. + /// + /// Use this when only specific PBS seats fire win/billing notifications + /// internally. The global [`suppress_nurl`](Self::suppress_nurl) switch still + /// suppresses every bidder when set. + #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] + pub suppress_nurl_bidders: Vec, } impl IntegrationConfig for PrebidIntegrationConfig { @@ -1341,6 +1349,15 @@ impl PrebidAuctionProvider { } } + fn should_suppress_bid_notifications(&self, bidder: &str) -> bool { + self.config.suppress_nurl + || self + .config + .suppress_nurl_bidders + .iter() + .any(|suppressed_bidder| suppressed_bidder == bidder) + } + /// Parse a single bid from `OpenRTB` response. fn parse_bid(&self, bid_obj: &Json, seat: &str) -> Result { let slot_id = bid_obj @@ -1370,7 +1387,8 @@ impl PrebidAuctionProvider { .and_then(|v| u32::try_from(v).ok()) .unwrap_or(0); - let nurl = if self.config.suppress_nurl { + let suppress_bid_notifications = self.should_suppress_bid_notifications(seat); + let nurl = if suppress_bid_notifications { None } else { bid_obj @@ -1379,7 +1397,7 @@ impl PrebidAuctionProvider { .map(std::string::ToString::to_string) }; - let burl = if self.config.suppress_nurl { + let burl = if suppress_bid_notifications { None } else { bid_obj @@ -1761,6 +1779,7 @@ mod tests { bid_param_override_rules: Vec::new(), consent_forwarding: ConsentForwardingMode::Both, suppress_nurl: false, + suppress_nurl_bidders: Vec::new(), } } @@ -4844,6 +4863,49 @@ set = { networkId = 42 } ); } + #[test] + fn parse_bid_strips_nurl_and_burl_for_configured_suppressed_bidder_only() { + let bid_json = serde_json::json!({ + "impid": "atf_sidebar_ad", + "price": 1.50, + "w": 300, + "h": 250, + "nurl": "https://ssp.example/win?id=abc123", + "burl": "https://ssp.example/bill?id=abc123" + }); + let config = PrebidIntegrationConfig { + suppress_nurl_bidders: vec!["appnexus".to_string()], + ..base_config() + }; + let provider = PrebidAuctionProvider::new(config); + + let suppressed_bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse suppressed bidder bid"); + let preserved_bid = provider + .parse_bid(&bid_json, "openx") + .expect("should parse unsuppressed bidder bid"); + + assert_eq!( + suppressed_bid.nurl, None, + "should strip nurl only for the configured bidder" + ); + assert_eq!( + suppressed_bid.burl, None, + "should strip burl only for the configured bidder" + ); + assert_eq!( + preserved_bid.nurl.as_deref(), + Some("https://ssp.example/win?id=abc123"), + "should preserve nurl for bidders not configured for suppression" + ); + assert_eq!( + preserved_bid.burl.as_deref(), + Some("https://ssp.example/bill?id=abc123"), + "should preserve burl for bidders not configured for suppression" + ); + } + #[test] fn parse_bid_preserves_ad_id_alongside_cache_id() { let bid_json = serde_json::json!({ diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 9863de1c7..a7d93d1a8 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -726,6 +726,8 @@ apply when the integration section exists in `trusted-server.toml`. | `bid_param_overrides` | Table | `{}` | Static per-bidder param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_zone_overrides` | Table | `{}` | Per-bidder, per-zone param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_override_rules` | Array[Table] | `[]` | Canonical ordered override rules with `when` matchers and `set` objects; evaluated after compatibility fields so later rules win on conflicts | +| `suppress_nurl` | Boolean | `false` | Strip `nurl` and `burl` from every PBS bid when the PBS deployment fires win/billing notifications server-side | +| `suppress_nurl_bidders` | Array[String] | `[]` | Bidder seats whose `nurl` and `burl` should be stripped while preserving client-side win/billing pixels for other bidders | | `debug` | Boolean | `false` | Enable debug mode (sets `ext.prebid.debug` and `returnallbidstatus`; surfaces debug metadata in responses) | | `test_mode` | Boolean | `false` | Set OpenRTB `test: 1` flag for non-billable test traffic (independent of `debug`) | | `debug_query_params` | String | `None` | Extra query params appended for debugging | diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index 42e0d6b7a..1e8870051 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -58,6 +58,8 @@ set = { placementId = "_s2sHeaderPlacement" } | `bid_param_overrides` | Table | `{}` | Static per-bidder param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_zone_overrides` | Table | `{}` | Per-bidder, per-zone param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_override_rules` | Array[Table] | `[]` | Canonical ordered override rules with `when` matchers and `set` objects; evaluated after compatibility fields so later rules win on conflicts | +| `suppress_nurl` | Boolean | `false` | Strip `nurl` and `burl` from every PBS bid when the PBS deployment fires win/billing notifications server-side | +| `suppress_nurl_bidders` | Array[String] | `[]` | Bidder seats whose `nurl` and `burl` should be stripped while preserving client-side win/billing pixels for other bidders | | `debug` | Boolean | `false` | Enable Prebid debug mode (sets `ext.prebid.debug` and `ext.prebid.returnallbidstatus`; surfaces debug metadata in auction responses) | | `test_mode` | Boolean | `false` | Set the OpenRTB `test: 1` flag so bidders treat the auction as non-billable test traffic. Separate from `debug` to avoid suppressing real demand | | `debug_query_params` | String | `None` | Extra query params appended for debugging | diff --git a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md index bdf24ff9c..8617ef877 100644 --- a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md +++ b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md @@ -487,10 +487,11 @@ The `hb_adid` match confirms two things: that the slot was filled (`!event.isEmp **and** that **our** Prebid bid (not a direct deal or backfill) won the GAM line item match. Only then are SSP win/billing pixels fired. -**Per-bidder suppression** (`[integrations.].suppress_nurl`, default `false`) -is retained as an escape hatch in case a specific PBS deployment fires `nurl` -internally and wants to avoid double-firing. APS `burl` follows the same client-side -path. +**Per-bidder suppression** (`[integrations.prebid].suppress_nurl_bidders`, default +`[]`) is retained as an escape hatch in case a specific PBS seat fires `nurl` +internally and wants to avoid double-firing. `[integrations.prebid].suppress_nurl = +true` remains a deployment-wide compatibility switch. APS `burl` follows the same +client-side path. > **Operational note:** Client-side firing introduces a small (~50–200ms) delay in > win-pixel arrival vs server-side firing. SSPs accept this — it's identical to @@ -1047,7 +1048,7 @@ saving. synchronous bid read, `slotRenderEnded` nurl + burl firing, `ts_initial` sentinel; add lazy slim-Prebid loader scheduled for post-`window.load` - **`crates/trusted-server-core/src/integrations/prebid.rs`** — add - `suppress_nurl` per-bidder config (default `false`); **no server-side nurl firing + `suppress_nurl_bidders` per-bidder config (default `[]`); **no server-side nurl firing in the page-load path** (firing is client-side from `slotRenderEnded`) - **`trusted-server.toml`** — add `[creative_opportunities]` section - **`crates/trusted-server-core/src/settings.rs`** — add `CreativeOpportunitiesConfig` diff --git a/trusted-server.toml b/trusted-server.toml index e1c35e11d..efff74693 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -95,6 +95,8 @@ client_side_bidders = [] # Set to true if PBS is configured to fire win/billing notifications server-side # (ext.prebid.events.enabled), to prevent the client from double-firing nurl/burl. # suppress_nurl = false +# For per-bidder suppression, list PBS seats that fire win/billing internally. +# suppress_nurl_bidders = ["exampleBidder"] [integrations.nextjs] enabled = false From 80fe39220cbd2eed458ca3471331b183b51bd186 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 10 Jun 2026 19:19:30 +0530 Subject: [PATCH 084/395] Resolve server-side ad template review issues --- .env.example | 2 +- crates/js/lib/src/core/types.ts | 2 + crates/js/lib/src/integrations/gpt/index.ts | 39 +++++- .../js/lib/src/integrations/prebid/index.ts | 39 ++++-- .../lib/test/integrations/gpt/index.test.ts | 97 ++++++++++++++ .../test/integrations/prebid/index.test.ts | 80 ++++++++++++ crates/trusted-server-core/src/publisher.rs | 123 ++++++++++++++++-- docs/guide/auction-orchestration.md | 20 +-- docs/guide/configuration.md | 6 +- trusted-server.toml | 69 ++-------- 10 files changed, 374 insertions(+), 103 deletions(-) diff --git a/.env.example b/.env.example index cec5d91de..c2ac88e3a 100644 --- a/.env.example +++ b/.env.example @@ -37,7 +37,7 @@ TRUSTED_SERVER__REQUEST_SIGNING__ENABLED=false # Prebid TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid-server.com/openrtb2/auction +# TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid-server.example.com/openrtb2/auction # TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=1000 # TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS=kargo,rubicon,appnexus # TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDES='{"bidder-name":{"param1":12345,"param2":"value"}}' diff --git a/crates/js/lib/src/core/types.ts b/crates/js/lib/src/core/types.ts index 31f66d0a7..57a14f3ec 100644 --- a/crates/js/lib/src/core/types.ts +++ b/crates/js/lib/src/core/types.ts @@ -98,6 +98,8 @@ export interface TsjsApi { servicesEnabled?: boolean; /** Maps actualDivId → slotId for slotRenderEnded billing lookup. */ divToSlotId?: Record; + /** Slot-level GPT targeting keys TS applied on the previous route. */ + prevSlotTargetingKeys?: Record; /** Guards SPA pushState hook installation. */ spaHookInstalled?: boolean; } diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index b7a81bc98..21f1f120d 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -25,6 +25,14 @@ import { installGptGuard } from './script_guard'; */ const TS_INITIAL_TARGETING_KEY = 'ts_initial' as const; +const TS_BID_TARGETING_KEYS = [ + 'hb_pb', + 'hb_bidder', + 'hb_adid', + 'hb_cache_host', + 'hb_cache_path', +] as const; +const TS_BASE_TARGETING_KEYS = [...TS_BID_TARGETING_KEYS, TS_INITIAL_TARGETING_KEY] as const; // ------------------------------------------------------------------ // googletag type stubs (minimal surface needed by the shim) @@ -34,6 +42,7 @@ interface GoogleTagSlot { getAdUnitPath(): string; getSlotElementId(): string; setTargeting(key: string, value: string | string[]): GoogleTagSlot; + clearTargeting?(key?: string): GoogleTagSlot; addService(service: GoogleTagPubAdsService): GoogleTagSlot; getTargeting?(key: string): string[]; } @@ -82,6 +91,14 @@ function messageSourceBelongsToConfiguredSlot(source: MessageEventSource | null) ); } +function clearTargetingKeys(slot: GoogleTagSlot, keys: Iterable): void { + if (typeof slot.clearTargeting !== 'function') return; + + for (const key of new Set(keys)) { + slot.clearTargeting(key); + } +} + interface GoogleTagPubAdsService { setTargeting(key: string, value: string | string[]): GoogleTagPubAdsService; getTargeting(key: string): string[]; @@ -333,6 +350,8 @@ export function installTsAdInit(): void { // All slots to refresh (TS-defined + publisher-owned reused). const slotsToRefresh: GoogleTagSlot[] = []; const divToSlotId: Record = {}; + const prevSlotTargetingKeys = ts.prevSlotTargetingKeys ?? {}; + const nextSlotTargetingKeys: Record = {}; slots.forEach((slot) => { // Resolve actual div ID: exact match first, then prefix query. @@ -363,19 +382,26 @@ export function installTsAdInit(): void { tsOwned = true; } + const slotDivId2 = gptSlot.getSlotElementId?.() ?? actualDivId; + clearTargetingKeys(gptSlot, [ + ...TS_BASE_TARGETING_KEYS, + ...(prevSlotTargetingKeys[actualDivId] ?? []), + ...(prevSlotTargetingKeys[slotDivId2] ?? []), + ]); + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v)); - (['hb_pb', 'hb_bidder', 'hb_adid', 'hb_cache_host', 'hb_cache_path'] as const).forEach( - (key) => { - if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); - } - ); + TS_BID_TARGETING_KEYS.forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); + }); gptSlot.setTargeting(TS_INITIAL_TARGETING_KEY, '1'); // Map both inner div and container div → slot ID so slotRenderEnded // (which reports the GPT slot's div, i.e. slotDivId/container) can look up // the slot, while adm injection (which targets the inner div) also works. divToSlotId[actualDivId] = slot.id; - const slotDivId2 = gptSlot.getSlotElementId?.() ?? actualDivId; if (slotDivId2 !== actualDivId) divToSlotId[slotDivId2] = slot.id; + const slotTargetingKeys = Object.keys(slot.targeting ?? {}); + nextSlotTargetingKeys[actualDivId] = slotTargetingKeys; + if (slotDivId2 !== actualDivId) nextSlotTargetingKeys[slotDivId2] = slotTargetingKeys; if (tsOwned) newSlots.push(gptSlot); slotsToRefresh.push(gptSlot); @@ -391,6 +417,7 @@ export function installTsAdInit(): void { ts.prevGptSlots = newSlots as unknown[]; // Replace (not merge) so destroyed slots from previous navigation don't linger. ts.divToSlotId = divToSlotId; + ts.prevSlotTargetingKeys = nextSlotTargetingKeys; // enableSingleRequest and enableServices must only be called once per page load. if (!ts.servicesEnabled) { diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index feb31e1a9..faa6ec04a 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -39,6 +39,14 @@ import { DEFAULT_PREBID_USER_ID_MODULES, PREBID_USER_ID_MODULE_REGISTRY } from ' const ADAPTER_CODE = 'trustedServer'; const BIDDER_PARAMS_KEY = 'bidderParams'; const ZONE_KEY = 'zone'; +const TS_REFRESH_TARGETING_KEYS = [ + 'ts_initial', + 'hb_pb', + 'hb_bidder', + 'hb_adid', + 'hb_cache_host', + 'hb_cache_path', +] as const; /** Configuration options for the Prebid integration. */ export interface PrebidNpmConfig { @@ -242,6 +250,7 @@ type PrebidUserIdEid = { type RefreshGptSlot = { getSlotElementId?: () => string; getTargeting?: (key: string) => string[]; + clearTargeting?: (key?: string) => RefreshGptSlot; getSizes?: () => unknown[]; }; @@ -333,6 +342,14 @@ function firstTargetingValue(values: string[] | undefined): string | undefined { return values?.find((value) => value.length > 0); } +function clearRefreshTargeting(slot: RefreshGptSlot): void { + if (typeof slot.clearTargeting !== 'function') return; + + for (const key of TS_REFRESH_TARGETING_KEYS) { + slot.clearTargeting(key); + } +} + function collectAuctionEids(): AuctionEid[] | undefined { if (typeof pbjs.getUserIdsAsEids !== 'function') { return undefined; @@ -569,8 +586,9 @@ export function installPrebidNpm(config?: Partial): typeof pbjs * Wraps `googletag.pubads().refresh()` so that when the publisher's GPT * refresh policy fires (sticky anchor, viewability dwell, infinite scroll), * Prebid runs a fresh client-side auction for the refreshing slots before - * the GAM call. TS-owned first-impression slots (`ts_initial=1`) are excluded - * — they are managed server-side and should not re-auction client-side. + * the GAM call. TS-owned first-impression slots (`ts_initial=1`) are included + * on later publisher refreshes, but stale TS server-side targeting is cleared + * before fresh Prebid targeting is applied. * * Must be called after `installPrebidNpm()` and after GPT is loaded. * Idempotent: safe to call multiple times — wraps only once via a sentinel. @@ -598,24 +616,20 @@ export function installRefreshHandler(timeoutMs = 1500): void { const originalRefresh = pubads.refresh.bind(pubads); pubads.refresh = function (slots?: unknown[], opts?: unknown) { // For bare refresh() calls (no slots arg), get all registered slots from GPT - // so we can filter out TS first-impression slots and auction the rest. + // so we can auction the same concrete slot list and avoid stale targeting. const targetSlots = ( slots ?? (pubads as { getSlots?: () => unknown[] }).getSlots?.() ?? [] ).filter((slot): slot is RefreshGptSlot => typeof slot === 'object' && slot !== null); - // Filter out TS first-impression slots — they don't need client-side refresh auctions. - const nonTsSlots = targetSlots.filter( - (slot) => !slot.getTargeting?.('ts_initial')?.includes('1') - ); - - if (!nonTsSlots.length) { - // All slots are TS-owned — pass through unchanged. + if (!targetSlots.length) { return originalRefresh(slots, opts); } - const adUnits = nonTsSlots.map((slot) => { + targetSlots.forEach(clearRefreshTargeting); + + const adUnits = targetSlots.map((slot) => { const injectedSlot = findInjectedSlotForRefresh(slot); const zone = injectedSlot?.targeting?.[ZONE_KEY] ?? firstTargetingValue(slot.getTargeting?.(ZONE_KEY)); @@ -638,8 +652,7 @@ export function installRefreshHandler(timeoutMs = 1500): void { adUnits, bidsBackHandler: () => { pbjs.setTargetingForGPTAsync?.(); - // Refresh only the non-TS slots (pass explicit list so TS slots are not re-refreshed). - originalRefresh(nonTsSlots, opts); + originalRefresh(targetSlots, opts); }, timeout: timeoutMs, }); diff --git a/crates/js/lib/test/integrations/gpt/index.test.ts b/crates/js/lib/test/integrations/gpt/index.test.ts index 839b121d6..08cedfc36 100644 --- a/crates/js/lib/test/integrations/gpt/index.test.ts +++ b/crates/js/lib/test/integrations/gpt/index.test.ts @@ -216,6 +216,103 @@ describe('GPT – installSlimPrebidLoader', () => { }); }); +describe('GPT – installTsAdInit', () => { + beforeEach(() => { + document.body.innerHTML = ''; + delete (window as any).tsjs; + delete (window as any).googletag; + }); + + afterEach(() => { + document.body.innerHTML = ''; + delete (window as any).tsjs; + delete (window as any).googletag; + }); + + it('clears stale TS-managed targeting before applying a new route to a reused GPT slot', async () => { + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + const slotTargeting = new Map([ + ['hb_pb', ['1.20']], + ['hb_bidder', ['kargo']], + ['hb_adid', ['old-ad']], + ['hb_cache_host', ['cache.example.com']], + ['hb_cache_path', ['/cache']], + ['ts_initial', ['1']], + ['pos', ['old-pos']], + ]); + const gptSlot: any = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn((key: string) => slotTargeting.get(key) ?? []), + setTargeting: vi.fn((key: string, value: string | string[]) => { + slotTargeting.set(key, Array.isArray(value) ? value : [value]); + return gptSlot; + }), + clearTargeting: vi.fn((key?: string) => { + if (key) { + slotTargeting.delete(key); + } else { + slotTargeting.clear(); + } + return gptSlot; + }), + }; + const pubads = { + getSlots: vi.fn(() => [gptSlot]), + enableSingleRequest: vi.fn(), + addEventListener: vi.fn(), + refresh: vi.fn(), + }; + const cmd: Array<() => void> = []; + cmd.push = (...callbacks: Array<() => void>) => { + callbacks.forEach((callback) => callback()); + return cmd.length; + }; + + document.body.innerHTML = '
'; + (window as any).googletag = { + cmd, + pubads: () => pubads, + defineSlot: vi.fn(), + destroySlots: vi.fn(), + enableServices: vi.fn(), + }; + (window as any).tsjs = { + prevSlotTargetingKeys: { + 'div-ad-homepage-header': ['pos'], + }, + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + bids: {}, + }; + + installTsAdInit(); + (window as any).tsjs.adInit(); + + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_bidder'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_host'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('pos'); + expect(slotTargeting.get('hb_pb')).toBeUndefined(); + expect(slotTargeting.get('hb_bidder')).toBeUndefined(); + expect(slotTargeting.get('hb_adid')).toBeUndefined(); + expect(slotTargeting.get('hb_cache_host')).toBeUndefined(); + expect(slotTargeting.get('hb_cache_path')).toBeUndefined(); + expect(slotTargeting.get('pos')).toBeUndefined(); + expect(slotTargeting.get('zone')).toEqual(['homepage']); + expect(slotTargeting.get('ts_initial')).toEqual(['1']); + }); +}); + describe('GPT shim – runtime gating', () => { type GatedWindow = Window & { __tsjs_gpt_enabled?: boolean; diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index c79bfd080..18f8dd8cd 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -769,6 +769,7 @@ describe('prebid/installPrebidNpm with server-injected config', () => { describe('prebid/installRefreshHandler', () => { beforeEach(() => { vi.clearAllMocks(); + mockRequestBids.mockReset(); mockPbjs.requestBids = mockRequestBids; mockPbjs.adUnits = []; (window as any).tsjs = undefined; @@ -833,6 +834,85 @@ describe('prebid/installRefreshHandler', () => { }) ); }); + + it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { + const originalRefresh = vi.fn(); + const clearTargeting = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn((key: string) => { + if (key === 'ts_initial') return ['1']; + if (key === 'zone') return ['homepage']; + return []; + }), + getSizes: vi.fn(() => [ + { getWidth: () => 970, getHeight: () => 250 }, + { getWidth: () => 728, getHeight: () => 90 }, + ]), + clearTargeting, + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + const setTargetingForGPTAsync = vi.fn(); + (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [ + [970, 250], + [728, 90], + ], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + timeout: 750, + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + mediaTypes: { + banner: { + name: 'homepage', + sizes: [ + [970, 250], + [728, 90], + ], + }, + }, + bids: [{ bidder: 'trustedServer', params: { zone: 'homepage' } }], + }), + ], + }) + ); + expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(clearTargeting).toHaveBeenCalledWith('hb_bidder'); + expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(originalRefresh).not.toHaveBeenCalled(); + + const bidsBackHandler = mockRequestBids.mock.calls[0][0].bidsBackHandler; + bidsBackHandler(); + + expect(setTargetingForGPTAsync).toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); + }); }); describe('prebid/client-side bidders', () => { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index ef526cca0..1c4c7e02d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1034,10 +1034,7 @@ pub async fn handle_publisher_request( ); let consent_context = ec_context.consent().clone(); - let ec_id = ec_context - .ec_value() - .map(str::to_string) - .unwrap_or_default(); + let ec_id = ec_context.ec_value().filter(|_| ec_allowed); let cookie_jar = handle_request_cookies(&http_req)?; let geo = ec_context.geo_info().cloned(); @@ -1120,7 +1117,7 @@ pub async fn handle_publisher_request( }; let mut auction_request = build_auction_request( &slots_ctx, - &ec_id, + ec_id, &consent_context, &request_info, req.get_header_str("user-agent"), @@ -1129,7 +1126,11 @@ pub async fn handle_publisher_request( .as_ref() .and_then(|j| j.get(COOKIE_TS_EIDS)) .map(|c| c.value().to_owned()); - let client_eids = resolve_client_auction_eids(None, ts_eids_value.as_deref()); + let client_eids = if ec_id.is_some() { + resolve_client_auction_eids(None, ts_eids_value.as_deref()) + } else { + None + }; let kv_eids = resolve_auction_eids(kv, auction.registry, ec_context); let merged_eids = merge_auction_eids(client_eids, kv_eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); @@ -1316,7 +1317,7 @@ pub(crate) struct MatchedSlotsContext<'a> { /// Build an [`AuctionRequest`] from matched creative opportunity slots. pub(crate) fn build_auction_request( slots_ctx: &MatchedSlotsContext<'_>, - ec_id: &str, + ec_id: Option<&str>, consent_context: &crate::consent::ConsentContext, request_info: &crate::http_util::RequestInfo, user_agent: Option<&str>, @@ -1330,15 +1331,20 @@ pub(crate) fn build_auction_request( "{}://{}{}", request_info.scheme, request_info.host, slots_ctx.request_path ); + let ec_id = ec_id.filter(|id| !id.is_empty()); + let request_id = ec_id.map_or_else( + || format!("ts-req-{}", uuid::Uuid::new_v4().simple()), + |id| format!("ts-{id}"), + ); AuctionRequest { - id: format!("ts-{}", ec_id), + id: request_id, slots, publisher: PublisherInfo { domain: request_info.host.clone(), page_url: Some(page_url.clone()), }, user: UserInfo { - id: Some(ec_id.to_string()), + id: ec_id.map(str::to_string), consent: Some(consent_context.clone()), eids: None, }, @@ -1477,7 +1483,7 @@ pub(crate) fn build_bids_script(bid_map: &serde_json::Map should be infallible"); let escaped = html_escape_for_script(&json); format!( - "", + "", escaped ) } @@ -1592,7 +1598,7 @@ pub async fn handle_page_bids( EcContext::read_from_request(settings, &req).change_context(TrustedServerError::Proxy { message: "page-bids: failed to read EC context".to_string(), })?; - let ec_id = ec_ctx.ec_value().map(str::to_string).unwrap_or_default(); + let ec_id = ec_ctx.ec_value().filter(|_| ec_ctx.ec_allowed()); let consent_context = ec_ctx.consent().clone(); let geo = ec_ctx.geo_info().cloned(); let cookie_jar = handle_request_cookies(&http_req)?; @@ -1631,7 +1637,7 @@ pub async fn handle_page_bids( }; let mut auction_request = build_auction_request( &slots_ctx, - &ec_id, + ec_id, &consent_context, &request_info, req.get_header_str("user-agent"), @@ -1640,7 +1646,11 @@ pub async fn handle_page_bids( .as_ref() .and_then(|j| j.get(COOKIE_TS_EIDS)) .map(|c| c.value().to_owned()); - let client_eids = resolve_client_auction_eids(None, ts_eids_value.as_deref()); + let client_eids = if ec_id.is_some() { + resolve_client_auction_eids(None, ts_eids_value.as_deref()) + } else { + None + }; let kv_eids = resolve_auction_eids(kv, registry, &ec_ctx); let merged_eids = merge_auction_eids(client_eids, kv_eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); @@ -2922,12 +2932,15 @@ mod tests { #[cfg(test)] mod creative_opportunities_tests { use super::super::{ - build_ad_slots_script, build_bid_map, build_bids_script, html_escape_for_script, + build_ad_slots_script, build_auction_request, build_bid_map, build_bids_script, + html_escape_for_script, MatchedSlotsContext, }; use crate::auction::types::{Bid, MediaType}; + use crate::consent::ConsentContext; use crate::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunityFormat, CreativeOpportunitySlot, }; + use crate::http_util::RequestInfo; use crate::price_bucket::PriceGranularity; use std::collections::HashMap; @@ -3350,6 +3363,88 @@ mod tests { assert!(!inner.contains('>'), "no unescaped > in bids script"); } + #[test] + fn bids_script_calls_ad_init_without_retry_timer() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + + let script = build_bids_script(&map); + + assert!( + script.contains("window.tsjs.adInit"), + "should hand off bids to adInit" + ); + assert!( + !script.contains("setTimeout"), + "should not retry adInit on a timer" + ); + assert!( + !script.contains("prevGptSlots"), + "should not use TS-owned slots as adInit success signal" + ); + } + + #[test] + fn auction_request_without_ec_id_omits_user_id_and_uses_non_ec_request_id() { + let slot = make_slot(); + let slots = [slot]; + let slots_ctx = MatchedSlotsContext { + matched_slots: &slots, + request_path: "/2024/01/my-article/", + }; + let request_info = RequestInfo { + host: "publisher.example.com".to_string(), + scheme: "https".to_string(), + }; + + let request = build_auction_request( + &slots_ctx, + None, + &ConsentContext::default(), + &request_info, + Some("Mozilla/5.0"), + ); + + assert_eq!(request.user.id, None, "should not forward an EC user id"); + assert!( + request.id.starts_with("ts-req-"), + "should use a non-EC request id, got {}", + request.id + ); + } + + #[test] + fn auction_request_with_ec_id_sets_user_id_and_ec_request_id() { + let slot = make_slot(); + let slots = [slot]; + let slots_ctx = MatchedSlotsContext { + matched_slots: &slots, + request_path: "/2024/01/my-article/", + }; + let request_info = RequestInfo { + host: "publisher.example.com".to_string(), + scheme: "https".to_string(), + }; + + let request = build_auction_request( + &slots_ctx, + Some("ec-abc"), + &ConsentContext::default(), + &request_info, + Some("Mozilla/5.0"), + ); + + assert_eq!( + request.user.id.as_deref(), + Some("ec-abc"), + "should forward EC id when identity consent allows it" + ); + assert_eq!( + request.id, "ts-ec-abc", + "should preserve existing EC-derived request id when present" + ); + } + #[test] fn html_escape_encodes_special_chars() { assert_eq!( diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index 3a55bc3de..d75958812 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -373,8 +373,8 @@ This is why mediation is important when using APS: without a mediator, APS bids ```toml [integrations.aps] enabled = true -pub_id = "5128" -endpoint = "https://aax.amazon-adsystem.com/e/dtb/bid" +pub_id = "example-publisher" +endpoint = "https://aps.example.com/e/dtb/bid" timeout_ms = 800 ``` @@ -593,8 +593,8 @@ debug = false [integrations.aps] enabled = true -pub_id = "5128" -endpoint = "https://aax.amazon-adsystem.com/e/dtb/bid" +pub_id = "example-publisher" +endpoint = "https://aps.example.com/e/dtb/bid" timeout_ms = 800 [integrations.adserver_mock] @@ -629,12 +629,12 @@ price_floor = 0.50 #### `[integrations.aps]` -| Field | Type | Default | Description | -| ------------ | ------ | ------------------------------------------- | --------------------------- | -| `enabled` | bool | `false` | Enable APS provider | -| `pub_id` | string | — | APS publisher ID (required) | -| `endpoint` | string | `https://aax.amazon-adsystem.com/e/dtb/bid` | APS TAM endpoint | -| `timeout_ms` | u32 | `800` | Request timeout | +| Field | Type | Default | Description | +| ------------ | ------ | ----------------------------------- | --------------------------- | +| `enabled` | bool | `false` | Enable APS provider | +| `pub_id` | string | — | APS publisher ID (required) | +| `endpoint` | string | `https://aps.example.com/e/dtb/bid` | APS TAM endpoint | +| `timeout_ms` | u32 | `800` | Request timeout | #### `[integrations.adserver_mock]` diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index a7d93d1a8..aceec9fa6 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -85,7 +85,7 @@ secret_store_id = "01GYYY" [integrations.prebid] enabled = true -server_url = "https://prebid-server.com/openrtb2/auction" +server_url = "https://prebid-server.example.com/openrtb2/auction" timeout_ms = 1200 bidders = ["kargo", "appnexus", "openx"] client_side_bidders = ["rubicon"] @@ -901,8 +901,8 @@ timeout_ms = 2000 [integrations.aps] enabled = true -pub_id = "5128" -endpoint = "https://aax.amazon-adsystem.com/e/dtb/bid" +pub_id = "example-publisher" +endpoint = "https://aps.example.com/e/dtb/bid" [integrations.prebid] enabled = true diff --git a/trusted-server.toml b/trusted-server.toml index efff74693..b8d3c50b6 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -57,8 +57,8 @@ config_store_id = "" # set config/secret store ids for k secret_store_id = "" [integrations.prebid] -enabled = true -server_url = "http://68.183.113.79:8000" +enabled = false +server_url = "https://prebid-server.example.com/openrtb2/auction" timeout_ms = 1000 bidders = ["kargo", "appnexus", "openx"] debug = false @@ -215,8 +215,8 @@ rewrite_script = true # ] [auction] -enabled = true -providers = ["prebid", "aps"] +enabled = false +providers = ["prebid"] # mediator = "adserver_mock" timeout_ms = 2000 # override per-publisher via TRUSTED_SERVER__AUCTION__TIMEOUT_MS # Context keys the JS client is allowed to forward into auction requests. @@ -224,9 +224,9 @@ timeout_ms = 2000 # override per-publisher via TRUSTED_SERVER__AUCTION__TIMEOUT allowed_context_keys = ["permutive_segments"] [integrations.aps] -enabled = true -pub_id = "test-pub" -endpoint = "https://origin-mocktioneer.cdintel.com/e/dtb/bid" +enabled = false +pub_id = "example-publisher" +endpoint = "https://aps.example.com/e/dtb/bid" timeout_ms = 1000 # override per-publisher via TRUSTED_SERVER__INTEGRATIONS__APS__TIMEOUT_MS [integrations.google_tag_manager] @@ -235,8 +235,8 @@ container_id = "GTM-XXXXXX" # upstream_url = "https://www.googletagmanager.com" [integrations.adserver_mock] -enabled = true -endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" +enabled = false +endpoint = "https://mediator.example.com/adserver/mediate" timeout_ms = 1000 # Debug configuration (all flags default to false — do not enable in production) @@ -271,7 +271,7 @@ timeout_ms = 1000 permutive_segments = "permutive" [creative_opportunities] -gam_network_id = "88059007" +gam_network_id = "123456789" # FCP is not affected by this value — body content above has already # streamed and painted before the hold begins. What this caps is the slip on # DOMContentLoaded and window.load. Worst case: a cache-hit page where origin @@ -281,50 +281,7 @@ gam_network_id = "88059007" auction_timeout_ms = 1500 # override via TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__AUCTION_TIMEOUT_MS price_granularity = "dense" -# Slot templates — override entire array via: +# No slot templates are enabled in the checked-in default config. Add +# `[[creative_opportunities.slot]]` entries via private config or override the +# entire array via: # TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT='[{"id":"...","gam_unit_path":"...",...}]' - -[[creative_opportunities.slot]] -id = "atf_sidebar_ad" -gam_unit_path = "/a/b/news" -div_id = "div-ad-atf-sidebar" -page_patterns = ["/20**", "/news/**"] -formats = [{ width = 300, height = 250 }] -floor_price = 0.50 - -[creative_opportunities.slot.targeting] -pos = "atf" -zone = "atfSidebar" - -[creative_opportunities.slot.providers.aps] -slot_id = "aps-slot-atf-sidebar" - -[[creative_opportunities.slot]] -id = "homepage_header_ad" -gam_unit_path = "/a/b/homepage" -div_id = "div-ad-homepage-header" -page_patterns = ["/"] -formats = [{ width = 728, height = 90 }] -floor_price = 0.50 - -[creative_opportunities.slot.targeting] -pos = "atf" -zone = "header" - -[creative_opportunities.slot.providers.aps] -slot_id = "aps-slot-homepage-header" - -[[creative_opportunities.slot]] -id = "homepage_footer_ad" -gam_unit_path = "/a/b/homepage" -div_id = "div-ad-homepage-footer" -page_patterns = ["/"] -formats = [{ width = 728, height = 90 }, { width = 768, height = 66 }] -floor_price = 0.50 - -[creative_opportunities.slot.targeting] -pos = "btf" -zone = "fixedBottom" - -[creative_opportunities.slot.providers.aps] -slot_id = "aps-slot-homepage-footer" From d3d43bc9700a314c30daed037fb5a09a3e03548c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 12 Jun 2026 13:29:09 +0530 Subject: [PATCH 085/395] Resolve server-side ad template auction review findings - Fail closed on consent: add consent_allows_server_side_auction() helper requiring effective TCF/GPP Purpose 1 for GDPR and unknown jurisdictions (or any request carrying an EU TCF signal); used by both the publisher navigation auction and /__ts/page-bids - Pass the adapter's geo-aware EcContext into handle_page_bids so the jurisdiction decision sees real geo instead of always-unknown - Make [auction].enabled a real kill switch for the automatic publisher navigation ad stack and the /__ts/page-bids auction - Normalize Prebid server_url: use it as-is when it already ends with /openrtb2/auction, otherwise append the path (backward compatible) - Advertise the effective auction budget in provider payloads: PBS tmax and APS timeout now use the orchestrator-capped context.timeout_ms instead of raw provider config - Add a one-shot adInitRefreshInProgress bypass so slim-Prebid's refresh wrapper passes adInit()'s internal refresh straight to GPT instead of clearing server-side targeting with a duplicate client-side auction - Sweep stale TS targeting (hb_*, ts_initial, route keys) from all previously TS-touched GPT slots before applying a new SPA route - Map the GPT slot element ID (container div) in the inline bootstrap's divToSlotId so container-backed slots fire nurl/burl beacons --- crates/js/lib/src/core/types.ts | 7 + .../js/lib/src/integrations/gpt/index.test.ts | 92 ++++++ crates/js/lib/src/integrations/gpt/index.ts | 33 +- .../js/lib/src/integrations/prebid/index.ts | 9 + .../test/integrations/prebid/index.test.ts | 50 +++ .../trusted-server-adapter-fastly/src/main.rs | 9 +- crates/trusted-server-core/src/consent/mod.rs | 113 ++++++- .../src/integrations/aps.rs | 49 ++- .../src/integrations/gpt_bootstrap.js | 8 + .../src/integrations/prebid.rs | 96 +++++- crates/trusted-server-core/src/publisher.rs | 303 ++++++++++-------- 11 files changed, 615 insertions(+), 154 deletions(-) diff --git a/crates/js/lib/src/core/types.ts b/crates/js/lib/src/core/types.ts index 57a14f3ec..4fb99f3b4 100644 --- a/crates/js/lib/src/core/types.ts +++ b/crates/js/lib/src/core/types.ts @@ -100,6 +100,13 @@ export interface TsjsApi { divToSlotId?: Record; /** Slot-level GPT targeting keys TS applied on the previous route. */ prevSlotTargetingKeys?: Record; + /** + * One-shot bypass for the slim-Prebid refresh wrapper: true only while + * adInit() runs its internal refresh of server-side-targeted slots, so the + * wrapper passes that refresh straight to GPT instead of starting a + * client-side auction that would clear the just-applied TS targeting. + */ + adInitRefreshInProgress?: boolean; /** Guards SPA pushState hook installation. */ spaHookInstalled?: boolean; } diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/src/integrations/gpt/index.test.ts index 5573993b8..aaf2657b4 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/src/integrations/gpt/index.test.ts @@ -125,6 +125,98 @@ describe('installTsAdInit', () => { fetchSpy.mockRestore(); }); + it('sets adInitRefreshInProgress only for the duration of the internal refresh', async () => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + let flagDuringRefresh: boolean | undefined; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([]), + addEventListener: vi.fn(), + refresh: vi.fn(() => { + flagDuringRefresh = (window as TestWindow).tsjs!.adInitRefreshInProgress; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(mockPubads.refresh).toHaveBeenCalled(); + expect(flagDuringRefresh).toBe(true); + expect((window as TestWindow).tsjs!.adInitRefreshInProgress).toBe(false); + }); + + it('clears stale TS targeting from previously touched slots when the new route has no TS slots', async () => { + const clearTargeting = vi.fn().mockReturnThis(); + const staleSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + clearTargeting, + getSlotElementId: vi.fn().mockReturnValue('div-old-route'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([staleSlot]), + addEventListener: vi.fn(), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn(), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + // New route has no matching TS slots. + adSlots: [], + bids: {}, + // Previous route touched the publisher-owned slot on div-old-route. + divToSlotId: { 'div-old-route': 'old_slot' }, + prevSlotTargetingKeys: { 'div-old-route': ['pos'] }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(clearTargeting).toHaveBeenCalledWith('hb_bidder'); + expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(clearTargeting).toHaveBeenCalledWith('pos'); + expect(mockPubads.refresh).not.toHaveBeenCalled(); + expect((window as TestWindow).tsjs!.divToSlotId).toEqual({}); + expect((window as TestWindow).tsjs!.prevSlotTargetingKeys).toEqual({}); + }); + it('keeps the GAM path when debug adm is present', async () => { const slotEl = document.getElementById('div-atf-sidebar')!; const mockSlot = { diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 21f1f120d..9054d4c15 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -353,6 +353,27 @@ export function installTsAdInit(): void { const prevSlotTargetingKeys = ts.prevSlotTargetingKeys ?? {}; const nextSlotTargetingKeys: Record = {}; + // Clear TS-managed targeting from every previously TS-touched GPT slot + // before applying the current route. Without this sweep, navigating to a + // route with no matching TS slots (or one where a previously touched + // publisher-owned slot is absent from the new slot list) leaves stale + // hb_* / ts_initial / route targeting that later publisher refreshes + // would reuse. + const prevTouchedDivIds = new Set([ + ...Object.keys(prevSlotTargetingKeys), + ...Object.keys(ts.divToSlotId ?? {}), + ]); + if (prevTouchedDivIds.size > 0) { + (g.pubads!().getSlots?.() ?? []).forEach((gptSlot: GoogleTagSlot) => { + const elementId = gptSlot.getSlotElementId(); + if (!prevTouchedDivIds.has(elementId)) return; + clearTargetingKeys(gptSlot, [ + ...TS_BASE_TARGETING_KEYS, + ...(prevSlotTargetingKeys[elementId] ?? []), + ]); + }); + } + slots.forEach((slot) => { // Resolve actual div ID: exact match first, then prefix query. // div_id in config may be a stable prefix (e.g. "ad-header-0-") when @@ -451,7 +472,17 @@ export function installTsAdInit(): void { } if (slotsToRefresh.length > 0) { - g.pubads!().refresh(slotsToRefresh); + // One-shot bypass: this internal refresh delivers the just-applied + // server-side targeting to GAM. If slim-Prebid has wrapped refresh(), + // it must pass this call straight through — not clear the targeting + // and run a duplicate client-side auction. Later publisher-initiated + // refreshes of the same slots still go through the wrapper normally. + ts.adInitRefreshInProgress = true; + try { + g.pubads!().refresh(slotsToRefresh); + } finally { + ts.adInitRefreshInProgress = false; + } } }); }; diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index faa6ec04a..cd2ffd265 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -615,6 +615,15 @@ export function installRefreshHandler(timeoutMs = 1500): void { const originalRefresh = pubads.refresh.bind(pubads); pubads.refresh = function (slots?: unknown[], opts?: unknown) { + // One-shot bypass for adInit()'s internal refresh: that refresh delivers + // freshly applied server-side targeting to GAM and must not be turned + // into a client-side auction (which would clear the TS targeting). + // Publisher-initiated refreshes of the same slots are not flagged and + // still run a fresh client-side auction below. + if (window.tsjs?.adInitRefreshInProgress) { + return originalRefresh(slots, opts); + } + // For bare refresh() calls (no slots arg), get all registered slots from GPT // so we can auction the same concrete slot list and avoid stale targeting. const targetSlots = ( diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index 18f8dd8cd..31a922869 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -913,6 +913,56 @@ describe('prebid/installRefreshHandler', () => { expect(setTargetingForGPTAsync).toHaveBeenCalled(); expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); }); + + it('passes the adInit internal refresh straight to GPT without a client-side auction', () => { + const originalRefresh = vi.fn(); + const clearTargeting = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + clearTargeting, + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { adInitRefreshInProgress: true }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).not.toHaveBeenCalled(); + expect(clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); + }); + + it('runs a client-side auction for publisher refreshes after adInit completes', () => { + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + clearTargeting: vi.fn(), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { adInitRefreshInProgress: false }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).toHaveBeenCalled(); + expect(originalRefresh).not.toHaveBeenCalled(); + }); }); describe('prebid/client-side bidders', () => { diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 884f2c15e..9a0f65c81 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -392,11 +392,14 @@ async fn route_request( (Method::GET, "/__ts/page-bids") => ( handle_page_bids( settings, - orchestrator, runtime_services, kv_graph.as_ref(), - registry_ref, - slots, + trusted_server_core::publisher::AuctionDispatch { + orchestrator, + slots, + registry: registry_ref, + }, + &ec_context, req, ) .await, diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index cd73acc9e..fad04d6b9 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -291,6 +291,26 @@ fn effective_tcf(ctx: &ConsentContext) -> Option<&types::TcfConsent> { .or_else(|| ctx.gpp.as_ref().and_then(|g| g.eu_tcf.as_ref())) } +/// Returns whether a server-side auction may be dispatched for this request. +/// +/// Fails closed for GDPR-relevant traffic: when an EU TCF signal is present +/// (`gdpr_applies`) **or** the request's geo jurisdiction is GDPR or unknown, +/// the effective TCF consent (standalone TC string or GPP EU TCF section) +/// must grant Purpose 1 (storage/access). Only requests from a known +/// non-GDPR jurisdiction with no EU TCF signal are freely allowed. +#[must_use] +pub fn consent_allows_server_side_auction(ctx: &ConsentContext) -> bool { + let requires_tcf_purpose1 = ctx.gdpr_applies + || matches!( + ctx.jurisdiction, + jurisdiction::Jurisdiction::Gdpr | jurisdiction::Jurisdiction::Unknown + ); + if !requires_tcf_purpose1 { + return true; + } + effective_tcf(ctx).is_some_and(|tcf| tcf.has_purpose_consent(1)) +} + /// Returns whether TCF consent allows EID transmission. #[must_use] fn allows_eid_transmission(tcf: &types::TcfConsent) -> bool { @@ -644,8 +664,8 @@ mod tests { use super::{ allows_ec_creation, apply_expiration_check, apply_tcf_conflict_resolution, - build_consent_context, build_context_from_signals, has_explicit_ec_withdrawal, - ConsentPipelineInput, + build_consent_context, build_context_from_signals, consent_allows_server_side_auction, + has_explicit_ec_withdrawal, ConsentPipelineInput, }; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ @@ -746,6 +766,95 @@ mod tests { } } + #[test] + fn auction_allowed_for_known_non_gdpr_jurisdiction_without_tcf_signal() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + ..ConsentContext::default() + }; + + assert!( + consent_allows_server_side_auction(&ctx), + "known non-GDPR jurisdiction with no EU TCF signal should allow auction" + ); + } + + #[test] + fn auction_fails_closed_for_gdpr_jurisdiction_without_consent() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + ..ConsentContext::default() + }; + + assert!( + !consent_allows_server_side_auction(&ctx), + "GDPR jurisdiction without a TCF signal should fail closed" + ); + } + + #[test] + fn auction_fails_closed_for_unknown_jurisdiction_without_consent() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Unknown, + ..ConsentContext::default() + }; + + assert!( + !consent_allows_server_side_auction(&ctx), + "unknown jurisdiction without a TCF signal should fail closed" + ); + } + + #[test] + fn auction_fails_closed_when_tcf_signal_present_without_purpose1() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gdpr_applies: true, + tcf: Some(TcfBuilder::new().with_storage(false).build()), + ..ConsentContext::default() + }; + + assert!( + !consent_allows_server_side_auction(&ctx), + "EU TCF signal without Purpose 1 should block auction even outside GDPR geo" + ); + } + + #[test] + fn auction_allowed_for_gdpr_jurisdiction_with_purpose1_consent() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some(TcfBuilder::new().with_storage(true).build()), + ..ConsentContext::default() + }; + + assert!( + consent_allows_server_side_auction(&ctx), + "GDPR jurisdiction with Purpose 1 consent should allow auction" + ); + } + + #[test] + fn auction_allowed_with_purpose1_via_gpp_eu_tcf_section() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + gpp: Some(GppConsent { + version: 1, + section_ids: vec![2], + eu_tcf: Some(TcfBuilder::new().with_storage(true).build()), + us_sale_opt_out: None, + }), + ..ConsentContext::default() + }; + + assert!( + consent_allows_server_side_auction(&ctx), + "Purpose 1 granted via GPP EU TCF section should allow auction" + ); + } + #[test] fn missing_geo_keeps_unknown_jurisdiction_and_blocks_ec_creation() { let req = build_request(); diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 34e26ebea..b415e5c88 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -309,7 +309,15 @@ impl ApsAuctionProvider { /// creative-opportunity slot ID so the caller can remap bids in the response. /// Populates consent fields (GDPR, US Privacy, GPP) from the /// [`ConsentContext`](crate::consent::ConsentContext) attached to the request. - fn to_aps_request(&self, request: &AuctionRequest) -> (ApsBidRequest, HashMap) { + /// + /// `timeout_ms` is the effective auction budget for this provider (already + /// capped by the orchestrator) — advertised to APS so it never expects more + /// time than the edge will actually wait. + fn to_aps_request( + &self, + request: &AuctionRequest, + timeout_ms: u32, + ) -> (ApsBidRequest, HashMap) { let mut slot_id_map: HashMap = HashMap::new(); let slots: Vec = request .slots @@ -364,7 +372,7 @@ impl ApsAuctionProvider { slots, page_url: request.publisher.page_url.clone(), user_agent: request.device.as_ref().and_then(|d| d.user_agent.clone()), - timeout: Some(self.config.timeout_ms), + timeout: Some(timeout_ms), gdpr, us_privacy, gpp, @@ -539,7 +547,10 @@ impl AuctionProvider for ApsAuctionProvider { // Transform to APS format; store the APS-slot-ID → creative-slot-ID map so // parse_response can remap bids back to the creative opportunity slot ID. - let (aps_request, slot_id_map) = self.to_aps_request(request); + // `context.timeout_ms` is the effective budget the orchestrator granted + // this provider — the payload must advertise the same deadline the edge + // backend enforces below. + let (aps_request, slot_id_map) = self.to_aps_request(request, context.timeout_ms); *self .slot_id_map .lock() @@ -760,7 +771,7 @@ mod tests { let provider = ApsAuctionProvider::new(config); let auction_request = create_test_auction_request(); - let (aps_request, _slot_id_map) = provider.to_aps_request(&auction_request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&auction_request, 800); // Verify basic fields assert_eq!(aps_request.pub_id, "5128"); @@ -830,7 +841,7 @@ mod tests { context: HashMap::new(), }; - let (aps_request, slot_id_map) = provider.to_aps_request(&request); + let (aps_request, slot_id_map) = provider.to_aps_request(&request, 800); assert_eq!( aps_request.slots[0].slot_id, "aps-slot-atf-sidebar", "should send configured APS slot ID to APS" @@ -1069,6 +1080,28 @@ mod tests { assert!(!provider.supports_media_type(&MediaType::Native)); } + #[test] + fn aps_payload_timeout_uses_effective_auction_budget_not_provider_config() { + // Provider config says 1000ms but the auction budget grants only 500ms — + // the payload must advertise the tighter effective deadline. + let config = ApsConfig { + enabled: true, + pub_id: "5128".to_string(), + endpoint: default_endpoint(), + timeout_ms: 1000, + }; + let provider = ApsAuctionProvider::new(config); + let request = create_test_auction_request(); + + let (aps_request, _slot_id_map) = provider.to_aps_request(&request, 500); + + assert_eq!( + aps_request.timeout, + Some(500), + "should advertise the effective auction budget, not the provider config timeout" + ); + } + #[test] fn test_aps_request_includes_consent_fields() { use crate::consent::ConsentContext; @@ -1091,7 +1124,7 @@ mod tests { ..Default::default() }); - let (aps_request, _slot_id_map) = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request, 800); // Verify GDPR consent let gdpr = aps_request.gdpr.expect("should have gdpr"); @@ -1120,7 +1153,7 @@ mod tests { let provider = ApsAuctionProvider::new(config); let request = create_test_auction_request(); // consent is None - let (aps_request, _slot_id_map) = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request, 800); assert!(aps_request.gdpr.is_none()); assert!(aps_request.us_privacy.is_none()); @@ -1147,7 +1180,7 @@ mod tests { ..Default::default() }); - let (aps_request, _slot_id_map) = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request, 800); let json = serde_json::to_value(&aps_request).expect("should serialize"); // GDPR fields present diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 0c7ea0dd2..cd4b05d42 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -75,7 +75,15 @@ }); // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts s.setTargeting("ts_initial", "1"); + // Map both the inner div and the GPT slot's element ID (the + // "-container" div when TS defined the slot there) so slotRenderEnded + // — which reports the GPT slot element ID — can find the slot for + // nurl/burl beacon firing. divToSlotId[actualDivId] = slot.id; + var slotElementId = s.getSlotElementId(); + if (slotElementId && slotElementId !== actualDivId) { + divToSlotId[slotElementId] = slot.id; + } if (tsOwned) newSlots.push(s); slotsToRefresh.push(s); }); diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index b757a1bf6..d8e411aed 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -904,6 +904,21 @@ impl PrebidAuctionProvider { }) } + /// Returns the full Prebid Server `OpenRTB2` auction endpoint URL. + /// + /// Backward-compatible normalization: `server_url` may be configured as + /// either the PBS origin (path is appended here) or the full endpoint + /// already ending in `/openrtb2/auction` (used as-is, ignoring a trailing + /// slash) — both shapes produce the same request URL. + fn auction_endpoint_url(&self) -> String { + let base = self.config.server_url.trim_end_matches('/'); + if base.ends_with("/openrtb2/auction") { + base.to_string() + } else { + format!("{base}/openrtb2/auction") + } + } + /// Convert auction request to `OpenRTB` format with all enrichments. fn to_openrtb( &self, @@ -1168,7 +1183,12 @@ impl PrebidAuctionProvider { .get_header_str(header::REFERER) .map(std::string::ToString::to_string); - let tmax = to_openrtb_i32(self.config.timeout_ms, "tmax", "request"); + // Advertise the effective auction budget, not the raw provider config: + // the orchestrator caps `context.timeout_ms` to the remaining auction + // budget, and the edge backend stops waiting after that long. Telling + // PBS it has more time than the edge will wait turns partial bids into + // edge timeouts. + let tmax = to_openrtb_i32(context.timeout_ms, "tmax", "request"); OpenRtbRequest { id: Some(request.id.clone()), @@ -1622,8 +1642,8 @@ impl AuctionProvider for PrebidAuctionProvider { if log::log_enabled!(log::Level::Debug) { match serde_json::to_string_pretty(&openrtb) { Ok(json) => log::debug!( - "Prebid OpenRTB request to {}/openrtb2/auction:\n{}", - self.config.server_url, + "Prebid OpenRTB request to {}:\n{}", + self.auction_endpoint_url(), json ), Err(e) => { @@ -1633,10 +1653,7 @@ impl AuctionProvider for PrebidAuctionProvider { } // Create HTTP request - let mut pbs_req = Request::new( - Method::POST, - format!("{}/openrtb2/auction", self.config.server_url), - ); + let mut pbs_req = Request::new(Method::POST, self.auction_endpoint_url()); copy_request_headers( context.request, &mut pbs_req, @@ -3073,7 +3090,7 @@ server_url = "https://prebid.example" assert_eq!( openrtb.tmax, Some(1000), - "should set tmax from config timeout_ms" + "should set tmax from the effective auction context timeout" ); assert_eq!( openrtb.cur, @@ -3083,15 +3100,72 @@ server_url = "https://prebid.example" } #[test] - fn to_openrtb_omits_tmax_when_timeout_exceeds_i32_max() { + fn auction_endpoint_url_appends_path_to_base_origin() { + let provider = PrebidAuctionProvider::new(base_config()); + assert_eq!( + provider.auction_endpoint_url(), + "https://prebid.example/openrtb2/auction", + "should append /openrtb2/auction to a base origin" + ); + } + + #[test] + fn auction_endpoint_url_does_not_double_append_full_endpoint() { + let mut config = base_config(); + config.server_url = "https://prebid.example/openrtb2/auction".to_string(); + let provider = PrebidAuctionProvider::new(config); + assert_eq!( + provider.auction_endpoint_url(), + "https://prebid.example/openrtb2/auction", + "should use a full endpoint URL as-is" + ); + let mut config = base_config(); - config.timeout_ms = i32::MAX as u32 + 1; + config.server_url = "https://prebid.example/openrtb2/auction/".to_string(); + let provider = PrebidAuctionProvider::new(config); + assert_eq!( + provider.auction_endpoint_url(), + "https://prebid.example/openrtb2/auction", + "should normalize a trailing slash on a full endpoint URL" + ); + } + + #[test] + fn to_openrtb_tmax_uses_effective_context_timeout_not_provider_config() { + // Provider config says 1000ms but the auction budget is only 500ms — + // PBS must be told the tighter effective deadline, otherwise the edge + // gives up before PBS responds. + let config = base_config(); + assert_eq!(config.timeout_ms, 1000, "should start from 1000ms config"); let provider = PrebidAuctionProvider::new(config); let auction_request = create_test_auction_request(); let settings = make_settings(); let request = Request::get("https://pub.example/auction"); - let context = create_test_auction_context(&settings, &request); + let context = shared_test_auction_context(&settings, &request, 500); + + let openrtb = provider.to_openrtb( + &auction_request, + &context, + None, + make_request_info(&context), + ); + + assert_eq!( + openrtb.tmax, + Some(500), + "should set tmax from the effective auction context timeout, not provider config" + ); + } + + #[test] + fn to_openrtb_omits_tmax_when_timeout_exceeds_i32_max() { + let provider = PrebidAuctionProvider::new(base_config()); + let auction_request = create_test_auction_request(); + + let settings = make_settings(); + let request = Request::get("https://pub.example/auction"); + let context = shared_test_auction_context(&settings, &request, i32::MAX as u32 + 1); let openrtb = provider.to_openrtb( &auction_request, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1c4c7e02d..eb2c44174 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -26,7 +26,7 @@ use crate::auction::types::{ }; use crate::backend::BackendConfig; use crate::compat; -use crate::consent::gate_eids_by_consent; +use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; use crate::ec::kv::KvIdentityGraph; @@ -577,6 +577,9 @@ pub(crate) fn is_prefetch_request(req: &Request) -> bool { /// Returns true only when the publisher request should run the full /// server-side ad stack: auction dispatch plus initial ad-slot injection. +/// +/// `auction_enabled` is the global `[auction].enabled` kill switch — when +/// false, no automatic server-side auction or ad-slot injection runs. pub(crate) fn should_run_server_side_ad_stack( is_get: bool, is_navigation: bool, @@ -584,6 +587,7 @@ pub(crate) fn should_run_server_side_ad_stack( is_bot: bool, has_matched_slots: bool, consent_allows_auction: bool, + auction_enabled: bool, ) -> bool { is_get && is_navigation @@ -591,6 +595,7 @@ pub(crate) fn should_run_server_side_ad_stack( && !is_bot && has_matched_slots && consent_allows_auction + && auction_enabled } /// Write winning bids from an auction result into the shared `ad_bids_state` lock. @@ -1067,13 +1072,10 @@ pub async fn handle_publisher_request( Vec::new() }; - // Non-GDPR regions (US, etc.) have no TCF string — auction is freely allowed. - // GDPR regions require TCF Purpose 1 (storage/access) before firing. - let consent_allows_auction = !consent_context.gdpr_applies - || consent_context - .tcf - .as_ref() - .is_some_and(|tcf| tcf.has_purpose_consent(1)); + // Fail closed for GDPR-relevant traffic: GDPR/unknown jurisdictions and + // requests carrying an EU TCF signal require effective TCF Purpose 1 + // (storage/access) before firing. Known non-GDPR jurisdictions are free. + let consent_allows_auction = consent_allows_server_side_auction(&consent_context); let should_run_ad_stack = should_run_server_side_ad_stack( is_get, @@ -1082,6 +1084,7 @@ pub async fn handle_publisher_request( is_bot, !matched_slots.is_empty(), consent_allows_auction, + auction.orchestrator.is_enabled(), ); let should_run_auction = should_run_ad_stack; @@ -1567,11 +1570,10 @@ fn is_supported_content_encoding(encoding: &str) -> bool { /// Returns [`TrustedServerError`] if cookie parsing or EC ID generation fails. pub async fn handle_page_bids( settings: &Settings, - orchestrator: &AuctionOrchestrator, services: &RuntimeServices, kv: Option<&KvIdentityGraph>, - registry: Option<&PartnerRegistry>, - slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + auction: AuctionDispatch<'_>, + ec_context: &EcContext, req: Request, ) -> Result> { let Some(co_config) = &settings.creative_opportunities else { @@ -1586,28 +1588,23 @@ pub async fn handle_page_bids( .map(|(_, v)| v.into_owned()) .unwrap_or_else(|| "/".to_string()); - let matched_slots: Vec<_> = crate::creative_opportunities::match_slots(slots, &path_param) - .into_iter() - .cloned() - .collect(); + let matched_slots: Vec<_> = + crate::creative_opportunities::match_slots(auction.slots, &path_param) + .into_iter() + .cloned() + .collect(); let http_req = compat::from_fastly_headers_ref(&req); let request_info = crate::http_util::RequestInfo::from_request(&http_req, &services.client_info); - let ec_ctx = - EcContext::read_from_request(settings, &req).change_context(TrustedServerError::Proxy { - message: "page-bids: failed to read EC context".to_string(), - })?; - let ec_id = ec_ctx.ec_value().filter(|_| ec_ctx.ec_allowed()); - let consent_context = ec_ctx.consent().clone(); - let geo = ec_ctx.geo_info().cloned(); + let ec_id = ec_context.ec_value().filter(|_| ec_context.ec_allowed()); + let consent_context = ec_context.consent(); + let geo = ec_context.geo_info().cloned(); let cookie_jar = handle_request_cookies(&http_req)?; - let consent_allows_auction = !consent_context.gdpr_applies - || consent_context - .tcf - .as_ref() - .is_some_and(|tcf| tcf.has_purpose_consent(1)); + // Same fail-closed jurisdiction-aware gate the publisher navigation path + // uses — relies on the adapter's geo-aware EC context. + let consent_allows_auction = consent_allows_server_side_auction(consent_context); // Same bot / prefetch guards the publisher path uses — without them this // endpoint would fire real SSP auctions on Sec-Purpose=prefetch warm-up @@ -1615,7 +1612,10 @@ pub async fn handle_page_bids( let is_prefetch = is_prefetch_request(&req); let is_bot = is_bot_user_agent(&req); - if matched_slots.is_empty() { + let auction_enabled = auction.orchestrator.is_enabled(); + if !auction_enabled { + log::debug!("page-bids: [auction].enabled is false — skipping auction"); + } else if matched_slots.is_empty() { log::debug!( "No creative opportunity slots matched path '{}' — skipping auction", path_param @@ -1629,69 +1629,74 @@ pub async fn handle_page_bids( ); } - let winning_bids = - if !matched_slots.is_empty() && consent_allows_auction && !is_bot && !is_prefetch { - let slots_ctx = MatchedSlotsContext { - matched_slots: &matched_slots, - request_path: &path_param, - }; - let mut auction_request = build_auction_request( - &slots_ctx, - ec_id, - &consent_context, - &request_info, - req.get_header_str("user-agent"), - ); - let ts_eids_value = cookie_jar - .as_ref() - .and_then(|j| j.get(COOKIE_TS_EIDS)) - .map(|c| c.value().to_owned()); - let client_eids = if ec_id.is_some() { - resolve_client_auction_eids(None, ts_eids_value.as_deref()) - } else { - None - }; - let kv_eids = resolve_auction_eids(kv, registry, &ec_ctx); - let merged_eids = merge_auction_eids(client_eids, kv_eids); - let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); - auction_request.user.eids = - gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); - if had_eids && auction_request.user.eids.is_none() { - log::warn!("Page-bids auction EIDs stripped by TCF consent gating"); - } - let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); - if client_ip.is_some() || geo.is_some() { - let device = auction_request.device.get_or_insert(DeviceInfo { - user_agent: None, - ip: None, - geo: None, - }); - device.ip = client_ip; - device.geo = geo.clone(); - } - let timeout_ms = co_config - .auction_timeout_ms - .unwrap_or(settings.auction.timeout_ms); - let auction_context = AuctionContext { - settings, - request: &req, - timeout_ms, - provider_responses: None, - services, - }; - match orchestrator - .run_auction(&auction_request, &auction_context) - .await - { - Ok(result) => result.winning_bids, - Err(e) => { - log::warn!("page-bids auction failed: {e:?}"); - std::collections::HashMap::new() - } - } + let winning_bids = if auction_enabled + && !matched_slots.is_empty() + && consent_allows_auction + && !is_bot + && !is_prefetch + { + let slots_ctx = MatchedSlotsContext { + matched_slots: &matched_slots, + request_path: &path_param, + }; + let mut auction_request = build_auction_request( + &slots_ctx, + ec_id, + consent_context, + &request_info, + req.get_header_str("user-agent"), + ); + let ts_eids_value = cookie_jar + .as_ref() + .and_then(|j| j.get(COOKIE_TS_EIDS)) + .map(|c| c.value().to_owned()); + let client_eids = if ec_id.is_some() { + resolve_client_auction_eids(None, ts_eids_value.as_deref()) } else { - std::collections::HashMap::new() + None + }; + let kv_eids = resolve_auction_eids(kv, auction.registry, ec_context); + let merged_eids = merge_auction_eids(client_eids, kv_eids); + let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); + auction_request.user.eids = + gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); + if had_eids && auction_request.user.eids.is_none() { + log::warn!("Page-bids auction EIDs stripped by TCF consent gating"); + } + let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); + if client_ip.is_some() || geo.is_some() { + let device = auction_request.device.get_or_insert(DeviceInfo { + user_agent: None, + ip: None, + geo: None, + }); + device.ip = client_ip; + device.geo = geo.clone(); + } + let timeout_ms = co_config + .auction_timeout_ms + .unwrap_or(settings.auction.timeout_ms); + let auction_context = AuctionContext { + settings, + request: &req, + timeout_ms, + provider_responses: None, + services, }; + match auction + .orchestrator + .run_auction(&auction_request, &auction_context) + .await + { + Ok(result) => result.winning_bids, + Err(e) => { + log::warn!("page-bids auction failed: {e:?}"); + std::collections::HashMap::new() + } + } + } else { + std::collections::HashMap::new() + }; let bid_map = build_bid_map( &winning_bids, @@ -1938,34 +1943,38 @@ mod tests { #[test] fn server_side_ad_stack_runs_only_when_all_auction_gates_pass() { assert!( - should_run_server_side_ad_stack(true, true, false, false, true, true), + should_run_server_side_ad_stack(true, true, false, false, true, true, true), "GET, real navigation, matched slots, and consent should run TS ad stack" ); assert!( - !should_run_server_side_ad_stack(false, true, false, false, true, true), + !should_run_server_side_ad_stack(false, true, false, false, true, true, true), "non-GET requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, false, false, false, true, true), + !should_run_server_side_ad_stack(true, false, false, false, true, true, true), "non-document requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, true, false, true, true), + !should_run_server_side_ad_stack(true, true, true, false, true, true, true), "prefetch requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, true, true, true), + !should_run_server_side_ad_stack(true, true, false, true, true, true, true), "bot requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, false, true), + !should_run_server_side_ad_stack(true, true, false, false, false, true, true), "requests with no matching slots should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, false), + !should_run_server_side_ad_stack(true, true, false, false, true, false, true), "requests without required consent should skip TS ad stack and injection" ); + assert!( + !should_run_server_side_ad_stack(true, true, false, false, true, true, false), + "disabled [auction].enabled kill switch should skip TS ad stack and injection" + ); } #[tokio::test] @@ -3501,12 +3510,46 @@ mod tests { fn settings_with_co() -> Settings { let toml = format!( - "{}\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") + } + + fn settings_with_co_auction_disabled() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = false\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", crate_test_settings_str() ); Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } + async fn run_page_bids( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> serde_json::Value { + let services = noop_services(); + let ec_context = + EcContext::read_from_request(settings, &req).expect("should read EC context"); + let response = handle_page_bids( + settings, + &services, + None, + AuctionDispatch { + orchestrator, + slots, + registry: None, + }, + &ec_context, + req, + ) + .await + .expect("should return ok response"); + serde_json::from_slice(&response.into_body_bytes()).expect("should be json") + } + fn article_slot() -> Vec { vec![CreativeOpportunitySlot { id: "atf".to_string(), @@ -3538,16 +3581,9 @@ mod tests { // all server-side auction activity and injection. let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let services = noop_services(); let req = make_page_bids_request("/2024/01/my-article/"); - let response = - handle_page_bids(&settings, &orchestrator, &services, None, None, &[], req) - .await - .expect("should return ok response"); - - let body: serde_json::Value = - serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + let body = run_page_bids(&settings, &orchestrator, &[], req).await; assert_eq!( body["slots"] @@ -3574,18 +3610,11 @@ mod tests { // for them. Same gate the publisher path applies. let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let services = noop_services(); let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1)"); - let response = - handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) - .await - .expect("should return ok response"); - - let body: serde_json::Value = - serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] @@ -3611,18 +3640,11 @@ mod tests { // SSP auctions — the user has not yet visited the page. let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let services = noop_services(); let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("sec-purpose", "prefetch"); - let response = - handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) - .await - .expect("should return ok response"); - - let body: serde_json::Value = - serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] @@ -3647,17 +3669,10 @@ mod tests { // Slots exist but request path does not match — no auction, no injection. let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let services = noop_services(); let slots = article_slot(); // slot matches /20** only let req = make_page_bids_request("/about"); // does not match - let response = - handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) - .await - .expect("should return ok response"); - - let body: serde_json::Value = - serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] @@ -3676,5 +3691,35 @@ mod tests { "non-matching URL should produce zero bids" ); } + + #[tokio::test] + async fn disabled_auction_returns_slots_but_no_bids() { + // [auction].enabled = false is a global kill switch: slot definitions + // are still returned (HTML structure unchanged) but no server-side + // auction may be dispatched. + let settings = settings_with_co_auction_disabled(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let req = make_page_bids_request("/2024/01/my-article/"); + + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 1, + "disabled auction should still return slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "disabled auction must not produce bids" + ); + } } } From 0f4dd86fefd0bcc164776a849963c0eb9516c0a3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 12 Jun 2026 13:55:28 +0530 Subject: [PATCH 086/395] Resolve beacon, validation, and orchestrator review findings - Dedupe win/billing beacons: fire each bid's nurl/burl at most once, keyed by slot + bid identity in shared tsjs state so the inline bootstrap and bundle listeners can never double-fire; unify the ourBidWon check (hb_adid confirmation with hb_bidder fallback for APS bids) across both implementations - Wire validate_slot_id into Settings::prepare_runtime so every load path (including env-injected slots on runtime-config adapters) rejects invalid slot IDs; build.rs settings stub gains a no-op - Normalize the client-controlled page-bids path parameter: strip query/fragment and force a leading slash before glob matching - Document the deliberate Cache-Control private, max-age=0 choice (BFCache eligibility per design spec section 4.7, not no-store) - Align orchestrator collect path with the parallel path: use parse_response_with_context for providers and the mediator, and add a defense-in-depth deadline check to the collect select-loop - Migrate adserver_mock off request-scoped Mutex state: the SSP bid index is rebuilt in parse_response_with_context from the context's provider responses; document why APS's slot_id_map cannot follow yet - Make platform_response_to_fastly infallible; drop the dead error arms - Remove redundant PriceGranularity::dense and MediaType::banner constructors in favor of Default-based serde field defaults - Clarify that the Prebid stored-request fallback cannot fire for the client /auction path (every ad unit carries a trustedServer entry) - Consolidate GPT JS suites under test/integrations/gpt/, replace the leaked module-scope addEventListener patch with a restored wrapper, and add installSpaAuctionHook coverage (pushState/replaceState/ popstate, stale-response guard, non-OK response, idempotence) --- crates/js/lib/src/core/types.ts | 6 + crates/js/lib/src/integrations/gpt/index.ts | 24 ++- .../integrations/gpt/ad_init.test.ts} | 95 ++++++---- .../test/integrations/gpt/spa_hook.test.ts | 167 ++++++++++++++++++ crates/trusted-server-core/build.rs | 8 + .../src/auction/orchestrator.rs | 157 ++++++++-------- .../trusted-server-core/src/auction/types.rs | 19 +- .../src/creative_opportunities.rs | 8 +- .../src/integrations/adserver_mock.rs | 147 ++++++++------- .../src/integrations/aps.rs | 5 + .../src/integrations/gpt_bootstrap.js | 15 +- .../src/integrations/prebid.rs | 7 + .../trusted-server-core/src/price_bucket.rs | 7 - crates/trusted-server-core/src/publisher.rs | 50 +++++- crates/trusted-server-core/src/settings.rs | 45 ++++- 15 files changed, 545 insertions(+), 215 deletions(-) rename crates/js/lib/{src/integrations/gpt/index.test.ts => test/integrations/gpt/ad_init.test.ts} (89%) create mode 100644 crates/js/lib/test/integrations/gpt/spa_hook.test.ts diff --git a/crates/js/lib/src/core/types.ts b/crates/js/lib/src/core/types.ts index 4fb99f3b4..1bdf1057b 100644 --- a/crates/js/lib/src/core/types.ts +++ b/crates/js/lib/src/core/types.ts @@ -98,6 +98,12 @@ export interface TsjsApi { servicesEnabled?: boolean; /** Maps actualDivId → slotId for slotRenderEnded billing lookup. */ divToSlotId?: Record; + /** + * Win/billing beacons already fired, keyed by `slotId|bidIdentity`. + * Shared between the inline GPT bootstrap and the bundle listener so a + * bid's nurl/burl fire at most once even across GAM re-renders. + */ + firedBeacons?: Record; /** Slot-level GPT targeting keys TS applied on the previous route. */ prevSlotTargetingKeys?: Record; /** diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 9054d4c15..6d058a0c8 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -453,13 +453,27 @@ export function installTsAdInit(): void { // Read ts.bids live (not the snapshot above) so post-navigation bid data is used. const bid = (ts.bids ?? {})[slotId] ?? {}; // Compare hb_adid targeting to verify the specific creative won. + // APS bids carry no hb_adid — fall back to hb_bidder presence + // (same heuristic as the inline bootstrap) so APS wins still bill. const ourBidWon = !event.isEmpty && - !!bid.hb_adid && - event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid; - if (ourBidWon) { - if (bid.nurl) navigator.sendBeacon(bid.nurl); - if (bid.burl) navigator.sendBeacon(bid.burl); + (bid.hb_adid + ? event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + : !!bid.hb_bidder); + if (ourBidWon && (bid.nurl || bid.burl)) { + // Fire win/billing beacons at most once per bid: GAM re-renders + // (publisher refreshes, repeated slotRenderEnded for the same + // line item) must not re-bill. New auctions produce new bid + // identities, so post-navigation bids still fire. Keyed in + // shared tsjs state so the inline-bootstrap listener and this + // one can never double-fire the same bid. + const beaconKey = `${slotId}|${bid.hb_adid ?? bid.nurl ?? bid.burl ?? ''}`; + const fired = (ts.firedBeacons ??= {}); + if (!fired[beaconKey]) { + fired[beaconKey] = true; + if (bid.nurl) navigator.sendBeacon(bid.nurl); + if (bid.burl) navigator.sendBeacon(bid.burl); + } } // GAM interceptor (testing): when adm is present, replace the GAM creative. diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/test/integrations/gpt/ad_init.test.ts similarity index 89% rename from crates/js/lib/src/integrations/gpt/index.test.ts rename to crates/js/lib/test/integrations/gpt/ad_init.test.ts index aaf2657b4..147ecebf1 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/test/integrations/gpt/ad_init.test.ts @@ -1,23 +1,36 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vitest'; // Track every 'message' EventListener added to window across the entire test // file. This lets the installTsRenderBridge suite remove all accumulated -// handlers (registered by each vi.resetModules() + import('./index') in the -// installTsAdInit suite) before dispatching its own events. +// handlers (registered by each vi.resetModules() + module re-import in the +// installTsAdInit suite) before dispatching its own events. The spy is +// restored and remaining handlers are detached in the afterAll below so the +// patch never leaks past this file. const allMessageHandlers: EventListener[] = []; -const _origWindowAddEventListener = window.addEventListener.bind(window); -// eslint-disable-next-line @typescript-eslint/no-explicit-any -(window as any).addEventListener = function ( +const originalWindowAddEventListener = window.addEventListener.bind(window); +// Plain wrapper, deliberately not vi.spyOn: the render-bridge suite spies on +// window.addEventListener itself, and vi.spyOn on an already-spied method +// returns the same mock instance — its "original" would alias the inner +// implementation and recurse. +(window as { addEventListener: typeof window.addEventListener }).addEventListener = (( type: string, handler: EventListenerOrEventListenerObject, - options?: unknown -) { - if (type === 'message') { + options?: boolean | AddEventListenerOptions +) => { + if (type === 'message' && handler) { allMessageHandlers.push(handler as EventListener); } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return _origWindowAddEventListener(type, handler as EventListener, options as any); -}; + return originalWindowAddEventListener(type, handler, options); +}) as typeof window.addEventListener; + +afterAll(() => { + for (const handler of allMessageHandlers) { + window.removeEventListener('message', handler); + } + allMessageHandlers.length = 0; + (window as { addEventListener: typeof window.addEventListener }).addEventListener = + originalWindowAddEventListener; +}); interface SlotRenderEvent { isEmpty: boolean; @@ -109,7 +122,7 @@ describe('installTsAdInit', () => { const fetchSpy = vi.spyOn(global, 'fetch'); - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -161,7 +174,7 @@ describe('installTsAdInit', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -201,7 +214,7 @@ describe('installTsAdInit', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -259,7 +272,7 @@ describe('installTsAdInit', () => { }, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -317,7 +330,7 @@ describe('installTsAdInit', () => { }, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -326,10 +339,17 @@ describe('installTsAdInit', () => { expect(beaconSpy).toHaveBeenCalledWith('https://ssp/win'); expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill'); + expect(beaconSpy).toHaveBeenCalledTimes(2); + + // GAM re-rendering the same line item (same hb_adid) must not re-fire + // the same bid's win/billing beacons. + capturedListener!({ isEmpty: false, slot: mockSlot }); + expect(beaconSpy).toHaveBeenCalledTimes(2); + beaconSpy.mockRestore(); }); - it('does not fire beacons when a rendered bid has no hb_adid confirmation', async () => { + it('fires APS-style beacons once via hb_bidder fallback and dedupes repeat renders', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); let capturedListener: ((e: SlotRenderEvent) => void) | undefined; @@ -373,18 +393,27 @@ describe('installTsAdInit', () => { }, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); expect(capturedListener).toBeDefined(); - capturedListener!({ isEmpty: false, slot: mockSlot }); - - expect(beaconSpy).not.toHaveBeenCalled(); + // Empty render never fires. capturedListener!({ isEmpty: true, slot: mockSlot }); expect(beaconSpy).not.toHaveBeenCalled(); + // First real render fires both beacons via the hb_bidder fallback + // (APS bids carry no hb_adid to confirm against). + capturedListener!({ isEmpty: false, slot: mockSlot }); + expect(beaconSpy).toHaveBeenCalledWith('https://aps/win'); + expect(beaconSpy).toHaveBeenCalledWith('https://aps/bill'); + expect(beaconSpy).toHaveBeenCalledTimes(2); + + // Re-render of the same bid (publisher refresh) must not re-bill. + capturedListener!({ isEmpty: false, slot: mockSlot }); + expect(beaconSpy).toHaveBeenCalledTimes(2); + beaconSpy.mockRestore(); }); @@ -433,7 +462,7 @@ describe('installTsAdInit', () => { }, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); capturedListener!({ isEmpty: false, slot: mockSlotNoMatch }); @@ -485,7 +514,7 @@ describe('installTsAdInit', () => { }, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -533,7 +562,7 @@ describe('installTsAdInit', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -580,7 +609,7 @@ describe('installTsAdInit', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -624,7 +653,7 @@ describe('installTsAdInit', () => { bids: {}, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -667,7 +696,7 @@ describe('installTsAdInit', () => { bids: {}, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); expect(() => (window as TestWindow).tsjs!.adInit!()).not.toThrow(); @@ -746,7 +775,7 @@ describe('installTsRenderBridge', () => { origAdd(type, handler as EventListener, opts as any); } ); - await import('./index'); + await import('../../../src/integrations/gpt/index'); addSpy.mockRestore(); // Restore only addEventListener — fetchStub must stay stubbed expect(bridgeListener, 'bridge listener should be registered').toBeDefined(); @@ -816,7 +845,7 @@ describe('installTsRenderBridge', () => { origAdd(type, handler as EventListener, opts as any); } ); - await import('./index'); + await import('../../../src/integrations/gpt/index'); addSpy.mockRestore(); expect(bridgeListener, 'bridge listener should be registered').toBeDefined(); @@ -850,7 +879,7 @@ describe('installTsRenderBridge', () => { }); it('ignores message when adId does not match any TS bid', async () => { - await import('./index'); + await import('../../../src/integrations/gpt/index'); fetchStub.mockResolvedValue({ ok: true, text: () => Promise.resolve('') } as Response); window.dispatchEvent( @@ -865,7 +894,7 @@ describe('installTsRenderBridge', () => { }); it('ignores matching adId messages from outside configured slot iframes', async () => { - await import('./index'); + await import('../../../src/integrations/gpt/index'); fetchStub.mockResolvedValue({ ok: true, text: () => Promise.resolve('') } as Response); const foreignIframe = document.createElement('iframe'); @@ -890,7 +919,7 @@ describe('installTsRenderBridge', () => { }); it('ignores non-Prebid messages', async () => { - await import('./index'); + await import('../../../src/integrations/gpt/index'); window.dispatchEvent( new MessageEvent('message', { data: JSON.stringify({ message: 'Other' }) }) ); diff --git a/crates/js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/js/lib/test/integrations/gpt/spa_hook.test.ts new file mode 100644 index 000000000..5a72c56e8 --- /dev/null +++ b/crates/js/lib/test/integrations/gpt/spa_hook.test.ts @@ -0,0 +1,167 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +import type { TsjsApi } from '../../../src/core/types'; + +type TestWindow = Window & { + googletag?: unknown; + tsjs?: TsjsApi; +}; + +const originalPushState = history.pushState.bind(history); +const originalReplaceState = history.replaceState.bind(history); + +async function importGptModule() { + return import('../../../src/integrations/gpt/index'); +} + +/** Flush the microtask/timer queue so onNavigate's awaits settle. */ +async function flushAsync(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +describe('installSpaAuctionHook', () => { + let fetchStub: ReturnType; + + beforeEach(() => { + vi.resetModules(); + delete (window as TestWindow).tsjs; + // Restore unwrapped history methods so each module import wraps exactly + // once — without this, wrappers from prior imports accumulate. + history.pushState = originalPushState; + history.replaceState = originalReplaceState; + fetchStub = vi.fn(); + vi.stubGlobal('fetch', fetchStub); + }); + + afterEach(() => { + history.pushState = originalPushState; + history.replaceState = originalReplaceState; + // Reset jsdom location back to root for the next test. + originalReplaceState({}, '', '/'); + vi.unstubAllGlobals(); + }); + + it('fetches page-bids on pushState and applies slots/bids via adInit', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [{ id: 's1' }], bids: { s1: { hb_pb: '1.00' } } }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/next-page'); + await flushAsync(); + + expect(fetchStub).toHaveBeenCalledWith( + '/__ts/page-bids?path=%2Fnext-page', + expect.objectContaining({ credentials: 'include' }) + ); + expect(ts.adSlots).toEqual([{ id: 's1' }]); + expect(ts.bids).toEqual({ s1: { hb_pb: '1.00' } }); + expect(adInit).toHaveBeenCalledTimes(1); + }); + + it('does not fetch when pushState targets the current path', async () => { + await importGptModule(); + + history.pushState({}, '', '/'); + await flushAsync(); + + expect(fetchStub).not.toHaveBeenCalled(); + }); + + it('fetches on replaceState and popstate navigation', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + + history.replaceState({}, '', '/replaced'); + await flushAsync(); + expect(fetchStub).toHaveBeenCalledWith( + '/__ts/page-bids?path=%2Freplaced', + expect.objectContaining({ credentials: 'include' }) + ); + + window.dispatchEvent(new PopStateEvent('popstate')); + await flushAsync(); + expect(fetchStub).toHaveBeenLastCalledWith( + '/__ts/page-bids?path=%2Freplaced', + expect.objectContaining({ credentials: 'include' }) + ); + }); + + it('drops a stale response that resolves after a newer navigation started', async () => { + let resolveFirst: ((value: unknown) => void) | undefined; + fetchStub + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFirst = resolve; + }) + ) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ slots: [{ id: 'newer' }], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/first'); + history.pushState({}, '', '/second'); + await flushAsync(); + + expect(ts.adSlots).toEqual([{ id: 'newer' }]); + expect(adInit).toHaveBeenCalledTimes(1); + + // First navigation's response arrives late — it must not overwrite the + // newer route's slots or trigger another adInit. + resolveFirst!({ + ok: true, + json: async () => ({ slots: [{ id: 'stale' }], bids: {} }), + }); + await flushAsync(); + + expect(ts.adSlots).toEqual([{ id: 'newer' }]); + expect(adInit).toHaveBeenCalledTimes(1); + }); + + it('leaves slots and bids untouched on a non-OK response', async () => { + fetchStub.mockResolvedValue({ ok: false, status: 500 }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + ts.adSlots = [{ id: 'existing' } as never]; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/error-page'); + await flushAsync(); + + expect(ts.adSlots).toEqual([{ id: 'existing' }]); + expect(adInit).not.toHaveBeenCalled(); + }); + + it('is idempotent — repeated install calls do not double-fetch a navigation', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + // Module init already installed the hook; both calls must be no-ops. + installSpaAuctionHook(); + installSpaAuctionHook(); + + history.pushState({}, '', '/once'); + await flushAsync(); + + expect(fetchStub).toHaveBeenCalledTimes(1); + }); +}); diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index a0cc07b30..fc5422af2 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -68,6 +68,14 @@ mod creative_opportunities { pub fn compile_slots(&mut self) {} } + /// Stub — the typed `slot` vec is always empty in the build context (see + /// `#[serde(skip)]` above), so `Settings::prepare_runtime` never reaches + /// this. Build-time slot-id validation happens in `main()` against + /// `slot_raw` instead. + pub fn validate_slot_id(_id: &str) -> Result<(), String> { + Ok(()) + } + fn default_price_granularity() -> String { "dense".to_string() } diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 8ccdca305..b77b110a9 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -813,37 +813,25 @@ impl AuctionOrchestrator { backend_to_provider.remove(&backend_name) { let response_time_ms = start_time.elapsed().as_millis() as u64; - match platform_response_to_fastly(platform_response) { - Ok(response) => { - match provider.parse_response(response, response_time_ms) { - Ok(auction_response) => { - log::info!( - "Provider '{}' returned {} bids ({}ms)", - auction_response.provider, - auction_response.bids.len(), - auction_response.response_time_ms - ); - responses.push(auction_response); - } - Err(e) => { - log::warn!( - "Provider '{}' parse failed: {:?}", - provider_name, - e - ); - responses.push(AuctionResponse::error( - &provider_name, - response_time_ms, - )); - } - } + let response = platform_response_to_fastly(platform_response); + // Mirror run_providers_parallel: use the context-aware + // parse so providers behave identically on both paths. + match provider.parse_response_with_context( + response, + response_time_ms, + context, + ) { + Ok(auction_response) => { + log::info!( + "Provider '{}' returned {} bids ({}ms)", + auction_response.provider, + auction_response.bids.len(), + auction_response.response_time_ms + ); + responses.push(auction_response); } Err(e) => { - log::warn!( - "Provider '{}' unsupported body: {:?}", - provider_name, - e - ); + log::warn!("Provider '{}' parse failed: {:?}", provider_name, e); responses .push(AuctionResponse::error(&provider_name, response_time_ms)); } @@ -859,6 +847,19 @@ impl AuctionOrchestrator { log::warn!("A provider request failed during collection: {:?}", e); } } + + // Defense-in-depth deadline guard, mirroring run_providers_parallel. + // Dispatch already caps each backend's first_byte_timeout at the + // remaining auction budget, so this should not fire in practice — + // it protects against the two paths drifting apart. + if remaining_budget_ms(auction_start, timeout_ms) == 0 && !remaining.is_empty() { + log::warn!( + "Auction timeout ({}ms) reached during collection, dropping {} remaining request(s)", + timeout_ms, + remaining.len() + ); + break; + } } let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { @@ -925,61 +926,47 @@ impl AuctionOrchestrator { ), }) { Ok(platform_resp) => { - match platform_response_to_fastly(platform_resp).change_context( - TrustedServerError::Auction { - message: format!( - "Mediator {} unsupported body", - mediator.provider_name() - ), - }, + let response = platform_response_to_fastly(platform_resp); + let response_time_ms = + mediator_start.elapsed().as_millis() as u64; + // Mirror run_parallel_mediation: use the + // context-aware parse so the mediator sees + // the collected provider responses. + match mediator.parse_response_with_context( + response, + response_time_ms, + &mediator_context, ) { - Ok(response) => { - let response_time_ms = - mediator_start.elapsed().as_millis() as u64; - match mediator - .parse_response(response, response_time_ms) - { - Ok(mediator_resp) => { - let winning = mediator_resp - .bids - .iter() - .filter_map(|bid| { - if bid.price.is_none() { - log::warn!( - "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", - mediator.provider_name(), - bid.slot_id - ); - None - } else { - Some((bid.slot_id.clone(), bid.clone())) - } - }) - .collect(); - let winning = self - .apply_floor_prices(winning, &floor_prices); - (Some(mediator_resp), winning) - } - Err(e) => { - log::warn!( - "Mediator '{}' parse failed: {:?}", - mediator.provider_name(), - e - ); - let winning = self.select_winning_bids( - &responses, - &floor_prices, - ); - (None, winning) - } - } + Ok(mediator_resp) => { + let winning = mediator_resp + .bids + .iter() + .filter_map(|bid| { + if bid.price.is_none() { + log::warn!( + "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", + mediator.provider_name(), + bid.slot_id + ); + None + } else { + Some((bid.slot_id.clone(), bid.clone())) + } + }) + .collect(); + let winning = + self.apply_floor_prices(winning, &floor_prices); + (Some(mediator_resp), winning) } Err(e) => { - log::warn!("Mediator body error: {:?}", e); - ( - None, - self.select_winning_bids(&responses, &floor_prices), - ) + log::warn!( + "Mediator '{}' parse failed: {:?}", + mediator.provider_name(), + e + ); + let winning = + self.select_winning_bids(&responses, &floor_prices); + (None, winning) } } } @@ -1063,12 +1050,8 @@ impl OrchestrationResult { } } -fn platform_response_to_fastly( - platform_response: PlatformResponse, -) -> Result> { - Ok(crate::compat::to_fastly_response( - platform_response.response, - )) +fn platform_response_to_fastly(platform_response: PlatformResponse) -> fastly::Response { + crate::compat::to_fastly_response(platform_response.response) } #[cfg(test)] diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 418da884d..2560ed92a 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -52,22 +52,15 @@ pub struct AdFormat { } /// Media type enumeration. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum MediaType { + #[default] Banner, Video, Native, } -impl MediaType { - /// Returns the Banner media type. - #[must_use] - pub fn banner() -> Self { - Self::Banner - } -} - /// Publisher information. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PublisherInfo { @@ -492,8 +485,12 @@ mod tests { } #[test] - fn media_type_banner_fn_returns_banner() { - assert_eq!(MediaType::banner(), MediaType::Banner); + fn media_type_defaults_to_banner() { + assert_eq!( + MediaType::default(), + MediaType::Banner, + "should default to Banner for serde field defaults" + ); } #[test] diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index a7b3d579a..645ba423f 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -36,8 +36,8 @@ pub struct CreativeOpportunitiesConfig { /// When absent, falls back to `[auction].timeout_ms` from global config. #[serde(default)] pub auction_timeout_ms: Option, - /// Price granularity for header-bidding price bucketing. - #[serde(default = "PriceGranularity::dense")] + /// Price granularity for header-bidding price bucketing. Defaults to `Dense`. + #[serde(default)] pub price_granularity: PriceGranularity, /// Slot templates. Empty vec = feature disabled (no auction fired, no globals injected). #[serde(default, deserialize_with = "vec_from_seq_or_map")] @@ -230,8 +230,8 @@ pub struct CreativeOpportunityFormat { pub width: u32, /// Creative height in pixels. pub height: u32, - /// Media type for this format. - #[serde(default = "MediaType::banner")] + /// Media type for this format. Defaults to `Banner`. + #[serde(default)] pub media_type: MediaType, } diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 8dc18e286..f1fd5ab89 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -10,7 +10,7 @@ use fastly::Request; use serde::{Deserialize, Serialize}; use serde_json::{json, Value as Json}; use std::collections::{BTreeMap, HashMap}; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use std::time::Duration; use validator::Validate; @@ -88,28 +88,42 @@ impl IntegrationConfig for AdServerMockConfig { // Provider // ============================================================================ -/// Lookup index built from original SSP bids during `request_bids`, consumed -/// during `parse_response` to restore render/accounting fields that the mock +/// Lookup index built from the original SSP bids, used while parsing the +/// mediation response to restore render/accounting fields that the mock /// mediator endpoint does not echo back. /// /// Keyed by `(provider_name, slot_id, bidder_name)`. type BidIndex = HashMap<(String, String, String), Bid>; +/// Builds the SSP-bid lookup index from the orchestrator-provided +/// bidder responses on the auction context. +fn build_bid_index(bidder_responses: &[AuctionResponse]) -> BidIndex { + let mut index = BidIndex::new(); + for response in bidder_responses { + for bid in &response.bids { + index.insert( + ( + response.provider.clone(), + bid.slot_id.clone(), + bid.bidder.clone(), + ), + bid.clone(), + ); + } + } + index +} + /// Mock ad server mediator provider. pub struct AdServerMockProvider { config: AdServerMockConfig, - /// Bridges SSP bid metadata from `request_bids` to `parse_response`. - bid_index: Mutex>, } impl AdServerMockProvider { /// Create a new mock ad server provider. #[must_use] pub fn new(config: AdServerMockConfig) -> Self { - Self { - config, - bid_index: Mutex::new(None), - } + Self { config } } /// Build the mediation endpoint URL, appending context values as query @@ -225,9 +239,10 @@ impl AdServerMockProvider { /// Parse `OpenRTB` response from mediation endpoint. /// Mediation returns decoded prices for all bids (including APS bids that were encoded). /// - /// `bid_index` is the SSP-bid lookup built in `request_bids`. The mock mediator - /// does not echo render/accounting fields back, so they are restored from the index - /// using `(seat, impid, bidder)` where bidder is recovered from the echoed `crid` + /// `bid_index` is the SSP-bid lookup built from the auction context's + /// bidder responses. The mock mediator does not echo render/accounting + /// fields back, so they are restored from the index using + /// `(seat, impid, bidder)` where bidder is recovered from the echoed `crid` /// field (`"{bidder}-creative"` format set during request construction). fn parse_mediation_response( &self, @@ -301,6 +316,45 @@ impl AdServerMockProvider { AuctionResponse::success("adserver_mock", all_bids, response_time_ms) } } + + /// Shared parse body for the context-aware and context-less trait methods. + /// + /// # Errors + /// + /// Returns an error when the mediation response body is not valid JSON. + fn parse_response_inner( + &self, + mut response: fastly::Response, + response_time_ms: u64, + bid_index: &BidIndex, + ) -> Result> { + if !response.get_status().is_success() { + log::warn!( + "AdServer Mock returned non-success: {}", + response.get_status() + ); + return Ok(AuctionResponse::error("adserver_mock", response_time_ms)); + } + + let body_bytes = response.take_body_bytes(); + let response_json: Json = + serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Auction { + message: "Failed to parse mediation response".to_string(), + })?; + + log::trace!("AdServer Mock response: {:?}", response_json); + + let auction_response = + self.parse_mediation_response(&response_json, response_time_ms, bid_index); + + log::info!( + "AdServer Mock returned {} bids in {}ms", + auction_response.bids.len(), + response_time_ms + ); + + Ok(auction_response) + } } impl AuctionProvider for AdServerMockProvider { @@ -322,23 +376,6 @@ impl AuctionProvider for AdServerMockProvider { bidder_responses.len() ); - // Build bid index so parse_response can restore nurl/burl/ad_id from - // the original SSP bids (the mock mediator does not echo these fields). - let mut index = BidIndex::new(); - for response in bidder_responses { - for bid in &response.bids { - index.insert( - ( - response.provider.clone(), - bid.slot_id.clone(), - bid.bidder.clone(), - ), - bid.clone(), - ); - } - } - *self.bid_index.lock().expect("should lock bid index") = Some(index); - // Build mediation request let mediation_req = self .build_mediation_request(request, bidder_responses) @@ -395,42 +432,28 @@ impl AuctionProvider for AdServerMockProvider { fn parse_response( &self, - mut response: fastly::Response, + response: fastly::Response, response_time_ms: u64, ) -> Result> { - if !response.get_status().is_success() { - log::warn!( - "AdServer Mock returned non-success: {}", - response.get_status() - ); - return Ok(AuctionResponse::error("adserver_mock", response_time_ms)); - } - - let body_bytes = response.take_body_bytes(); - let response_json: Json = - serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Auction { - message: "Failed to parse mediation response".to_string(), - })?; - - log::trace!("AdServer Mock response: {:?}", response_json); - - let bid_index = self - .bid_index - .lock() - .expect("should lock bid index") - .take() - .unwrap_or_default(); - - let auction_response = - self.parse_mediation_response(&response_json, response_time_ms, &bid_index); - - log::info!( - "AdServer Mock returned {} bids in {}ms", - auction_response.bids.len(), - response_time_ms - ); + // No auction context available — nurl/burl/ad_id restoration from the + // original SSP bids is skipped. The orchestrator always calls + // [`parse_response_with_context`], so this path only serves callers + // outside the orchestration flow. + log::debug!("adserver_mock: parsing without context — SSP bid metadata unavailable"); + self.parse_response_inner(response, response_time_ms, &BidIndex::new()) + } - Ok(auction_response) + fn parse_response_with_context( + &self, + response: fastly::Response, + response_time_ms: u64, + context: &AuctionContext<'_>, + ) -> Result> { + // Rebuild the SSP-bid lookup from the orchestrator-provided bidder + // responses so nurl/burl/ad_id survive mediation. Request-scoped data + // travels on the context instead of provider-instance state. + let bid_index = build_bid_index(context.provider_responses.unwrap_or(&[])); + self.parse_response_inner(response, response_time_ms, &bid_index) } fn supports_media_type(&self, media_type: &MediaType) -> bool { diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index b415e5c88..ed8c73354 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -290,6 +290,11 @@ pub struct ApsAuctionProvider { // Written by request_bids before the async send; read by parse_response when the // response arrives. Safe because Fastly Compute runs each request in an isolated // single-threaded Wasm instance — the Mutex never contends in practice. + // + // Unlike adserver_mock's bid index (rebuilt in parse_response_with_context + // from context.provider_responses), this map derives from the AuctionRequest, + // which AuctionContext does not carry — migrating it off provider-instance + // state needs the request threaded through the context first. slot_id_map: std::sync::Mutex>, } diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index cd4b05d42..341b376a6 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -103,9 +103,18 @@ (b.hb_adid ? ev.slot.getTargeting("hb_adid")[0] === b.hb_adid : !!b.hb_bidder); - if (ourBidWon) { - if (b.nurl) navigator.sendBeacon(b.nurl); - if (b.burl) navigator.sendBeacon(b.burl); + if (ourBidWon && (b.nurl || b.burl)) { + // Fire each bid's win/billing beacons at most once — GAM can + // re-render the same line item on publisher refreshes. Keep the + // key format in sync with the bundle listener in index.ts; the + // map lives on tsjs so both listeners share dedupe state. + var beaconKey = slotId + "|" + (b.hb_adid || b.nurl || b.burl || ""); + var fired = (ts.firedBeacons = ts.firedBeacons || {}); + if (!fired[beaconKey]) { + fired[beaconKey] = true; + if (b.nurl) navigator.sendBeacon(b.nurl); + if (b.burl) navigator.sendBeacon(b.burl); + } } }); } diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index d8e411aed..658307a92 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -984,6 +984,13 @@ impl PrebidAuctionProvider { // When no inline PBS bidder params exist (e.g. creative-opportunity slots // whose PBS params live in stored requests), tell PBS to resolve bidder // config from the stored request keyed by this slot ID. + // + // This cannot fire for the client /auction path: the JS adapter + // injects a `trustedServer` entry into every ad unit, so `bidder` + // is only empty for server-side creative-opportunity slots with + // no inline provider params (or when `config.bidders` is empty, + // where PBS previously received an empty bidder map and returned + // no bids — a stored-request miss is the same no-bid outcome). let storedrequest = if bidder.is_empty() { Some(ImpStoredRequest { id: slot.id.clone(), diff --git a/crates/trusted-server-core/src/price_bucket.rs b/crates/trusted-server-core/src/price_bucket.rs index cfdca9eb4..8fc4e50e6 100644 --- a/crates/trusted-server-core/src/price_bucket.rs +++ b/crates/trusted-server-core/src/price_bucket.rs @@ -11,13 +11,6 @@ pub enum PriceGranularity { Auto, } -impl PriceGranularity { - #[must_use] - pub fn dense() -> Self { - Self::Dense - } -} - #[must_use] pub fn price_bucket(cpm: f64, granularity: PriceGranularity) -> String { // Reject NaN / Inf early so the `(x * 100.0).floor() as u64` cast below diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index eb2c44174..027903893 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1208,7 +1208,10 @@ pub async fn handle_publisher_request( }; // §4.7: assembled HTML responses must never be shared-cached — per-user bid data - // travels inline. Apply regardless of slot match or auction outcome (§8). + // travels inline. `private, max-age=0` is deliberate (not `no-store`): it keeps + // the page BFCache-eligible while restricting reuse to the same user's browser + // with revalidation; `Surrogate-Control` removal handles the Fastly shared + // cache. Apply regardless of slot match or auction outcome (§8). let origin_content_type = response .get_header(header::CONTENT_TYPE) .and_then(|h| h.to_str().ok()) @@ -1559,6 +1562,20 @@ fn is_supported_content_encoding(encoding: &str) -> bool { matches!(encoding, "" | "identity" | "gzip" | "deflate" | "br") } +/// Normalizes the client-supplied `path` query parameter before glob matching. +/// +/// The SPA hook sends `location.pathname`, but the parameter is +/// client-controlled: strip any query string or fragment and force a leading +/// `/` so slot `page_patterns` always match against a canonical path shape. +fn normalize_page_bids_path(raw: &str) -> String { + let path = raw.split(['?', '#']).next().unwrap_or(""); + if path.starts_with('/') { + path.to_string() + } else { + format!("/{path}") + } +} + /// Handle `GET /__ts/page-bids?path=` — server-side auction for SPA navigation. /// /// Matches creative opportunity slots for the given path, runs a server-side @@ -1585,7 +1602,7 @@ pub async fn handle_page_bids( .get_url() .query_pairs() .find(|(k, _)| k == "path") - .map(|(_, v)| v.into_owned()) + .map(|(_, v)| normalize_page_bids_path(&v)) .unwrap_or_else(|| "/".to_string()); let matched_slots: Vec<_> = @@ -3692,6 +3709,35 @@ mod tests { ); } + #[test] + fn normalize_page_bids_path_strips_query_fragment_and_forces_leading_slash() { + assert_eq!( + normalize_page_bids_path("/2024/01/article/"), + "/2024/01/article/", + "canonical path should pass through unchanged" + ); + assert_eq!( + normalize_page_bids_path("/2024/01/article/?utm_source=x"), + "/2024/01/article/", + "query string should be stripped before glob matching" + ); + assert_eq!( + normalize_page_bids_path("/2024/01/article/#section"), + "/2024/01/article/", + "fragment should be stripped before glob matching" + ); + assert_eq!( + normalize_page_bids_path("2024/01/article/"), + "/2024/01/article/", + "missing leading slash should be added" + ); + assert_eq!( + normalize_page_bids_path(""), + "/", + "empty path should normalize to root" + ); + } + #[tokio::test] async fn disabled_auction_returns_slots_but_no_bids() { // [auction].enabled = false is a global kill switch: slot definitions diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 642ee4366..dc59bdfa9 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -776,7 +776,8 @@ impl Settings { /// /// # Errors /// - /// Returns a configuration error if any cached runtime artifact cannot be prepared. + /// Returns a configuration error if any cached runtime artifact cannot be + /// prepared, or if a creative opportunity slot has an invalid ID. pub fn prepare_runtime(&mut self) -> Result<(), Report> { for handler in &self.handlers { handler.prepare_runtime()?; @@ -784,6 +785,16 @@ impl Settings { if let Some(co) = &mut self.creative_opportunities { co.compile_slots(); + // Slot IDs flow into injected HTML/JS and provider payloads, and + // can arrive via TRUSTED_SERVER__ env overrides that bypass any + // static config review — validate them on every load path. + for slot in &co.slot { + crate::creative_opportunities::validate_slot_id(&slot.id).map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!("Invalid creative opportunity slot id: {err}"), + }) + })?; + } } Ok(()) @@ -2707,6 +2718,38 @@ auction_timeout_ms = 500 assert_eq!(co.auction_timeout_ms, Some(500)); } + #[test] + fn settings_rejects_invalid_creative_opportunity_slot_id() { + let toml = r#" +[[handlers]] +path = "^/_ts/admin" +username = "admin" +password = "unit-test-admin-secret" + +[publisher] +domain = "example.com" +cookie_domain = ".example.com" +origin_url = "https://origin.example.com" +proxy_secret = "secret" + +[ec] +passphrase = "test-secret-key-32-bytes-minimum" + +[creative_opportunities] +gam_network_id = "21765378893" + +[[creative_opportunities.slot]] +id = "xss"#; + let state = Arc::new(Mutex::new(Some(bids_script.to_string()))); + let params = OwnedProcessResponseParams { + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "Text/HTML; Charset=utf-8".to_string(), + ad_slots_script: Some( + r#""# + .to_string(), + ), + ad_bids_state: state, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let mut output = Vec::new(); + + stream_publisher_body( + Body::from(b"content".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect("should process mixed-case HTML content type"); + + let html = String::from_utf8(output).expect("should be valid UTF-8"); + assert!( + html.contains(".adSlots=JSON.parse"), + "mixed-case HTML must use the HTML processor and inject ad slots. Got: {html}" + ); + assert!( + html.contains(".bids=JSON.parse"), + "mixed-case HTML must use the HTML processor and inject bids. Got: {html}" + ); + } + /// Mid-stream decoder failure must surface as an error. The adapter /// relies on this: once headers are committed, it logs and drops the /// `StreamingBody` so the client sees a truncated response. If a decode From 8fb30b3ce7a5c2f4485882397e32f3fb6b0919ad Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 14 Jun 2026 10:43:51 +0530 Subject: [PATCH 096/395] Bind render bridge to source slot and fix refresh parity Resolve three #680 review findings on the server-side ad runtime: - Render bridge now requires the requesting iframe's slot to own the resolved hb_adid before responding or firing win/billing beacons. Previously an iframe under slot A could request slot B's adId and receive slot B's creative while firing slot B's beacons. - Refresh ad units now include configured client-side bidders by merging matching pbjs.adUnits bid entries, so native Prebid demand is not dropped on refresh/scroll impressions. - Inline GPT bootstrap wraps its internal refresh with the adInitRefreshInProgress sentinel, mirroring the TS adInit so a pre-installed slim-Prebid refresh wrapper does not clear TS targeting. Add regression tests for the two-slot render-bridge mismatch and the client-side bidder refresh merge. --- crates/js/lib/src/integrations/gpt/index.ts | 16 +++-- .../js/lib/src/integrations/prebid/index.ts | 38 +++++++++++- .../lib/test/integrations/gpt/ad_init.test.ts | 45 ++++++++++++++ .../test/integrations/prebid/index.test.ts | 58 +++++++++++++++++++ .../src/integrations/gpt_bootstrap.js | 12 +++- 5 files changed, 161 insertions(+), 8 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index eae2fc881..8d138a9bf 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -80,15 +80,15 @@ function candidateSlotRoots(divId: string): HTMLElement[] { return roots; } -function messageSourceBelongsToConfiguredSlot(source: MessageEventSource | null): boolean { - if (!source) return false; +function slotIdForMessageSource(source: MessageEventSource | null): string | undefined { + if (!source) return undefined; const slots = window.tsjs?.adSlots ?? []; - return slots.some((slot) => + return slots.find((slot) => candidateSlotRoots(slot.div_id).some((root) => Array.from(root.querySelectorAll('iframe')).some((iframe) => iframe.contentWindow === source) ) - ); + )?.id; } function clearTargetingKeys(slot: GoogleTagSlot, keys: Iterable): void { @@ -672,7 +672,8 @@ export function installTsRenderBridge(): void { const port = e.ports?.[0]; if (!port) return; - if (!messageSourceBelongsToConfiguredSlot(e.source)) return; + const sourceSlotId = slotIdForMessageSource(e.source); + if (!sourceSlotId) return; // Build reverse map adId → slotId from live window.tsjs.bids. const bids = window.tsjs?.bids ?? {}; @@ -689,6 +690,11 @@ export function installTsRenderBridge(): void { // Not a TS bid — let Prebid.js handle it. if (!slotId || !matchedBid) return; + // The requesting iframe's slot must own the resolved adId. Without this an + // iframe under slot A could request slot B's hb_adid and receive slot B's + // creative/dimensions while firing slot B's win/billing beacons. + if (slotId !== sourceSlotId) return; + const slot = window.tsjs?.adSlots?.find((s) => s.id === slotId); const [width, height] = slot?.formats?.[0] ?? [728, 90]; diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index d42c7d265..61b546e5b 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -342,6 +342,36 @@ function firstTargetingValue(values: string[] | undefined): string | undefined { return values?.find((value) => value.length > 0); } +/** + * Collect the configured client-side bidder entries for a refreshing slot. + * + * Synthetic refresh ad units carry only the `trustedServer` bid. The + * `requestBids` shim preserves a client-side bidder only when its bid entry is + * already present on the ad unit, so without re-attaching them here publishers + * that split demand between server-side and native Prebid adapters would lose + * all client-side demand on refresh/scroll impressions. Bids are sourced from + * the matching `pbjs.adUnits` entry (by ad unit code) so the publisher's + * configured params are preserved. + */ +function clientSideBidsForRefresh( + code: string +): Array<{ bidder: string; params: Record }> { + const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); + if (clientSideBidders.size === 0) return []; + + const adUnits = (pbjs.adUnits ?? []) as TrustedServerAdUnit[]; + const match = adUnits.find((unit) => unit.code === code); + if (!match?.bids) return []; + + const bids: Array<{ bidder: string; params: Record }> = []; + for (const bid of match.bids) { + if (bid?.bidder && clientSideBidders.has(bid.bidder)) { + bids.push({ bidder: bid.bidder, params: bid.params ?? {} }); + } + } + return bids; +} + function clearRefreshTargeting(slot: RefreshGptSlot): void { if (typeof slot.clearTargeting !== 'function') return; @@ -663,10 +693,14 @@ export function installRefreshHandler(timeoutMs = 1500): void { ...(zone ? { name: zone } : {}), }; + const code = refreshSlotElementId(slot) ?? 'refresh-slot'; return { - code: refreshSlotElementId(slot) ?? 'refresh-slot', + code, mediaTypes: { banner }, - bids: [{ bidder: ADAPTER_CODE, params: zone ? { [ZONE_KEY]: zone } : {} }], + bids: [ + { bidder: ADAPTER_CODE, params: zone ? { [ZONE_KEY]: zone } : {} }, + ...clientSideBidsForRefresh(code), + ], }; }); diff --git a/crates/js/lib/test/integrations/gpt/ad_init.test.ts b/crates/js/lib/test/integrations/gpt/ad_init.test.ts index 3a69c5a42..73d3be9c3 100644 --- a/crates/js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/js/lib/test/integrations/gpt/ad_init.test.ts @@ -932,6 +932,51 @@ describe('installTsRenderBridge', () => { foreignIframe.remove(); }); + it('ignores a request whose source slot does not own the resolved adId', async () => { + // Two configured slots; slot A's iframe requests slot B's hb_adid. The + // bridge must not return slot B's creative or fire slot B's beacons. + (window as TestWindow).tsjs.bids.homepage_footer = { + hb_adid: 'footer-uuid', + hb_bidder: 'kargo', + hb_pb: '2.00', + hb_cache_host: 'openads.example.com', + hb_cache_path: '/cache', + nurl: 'https://ssp.example/footer-win', + burl: 'https://ssp.example/footer-bill', + }; + (window as TestWindow).tsjs.adSlots.push({ + id: 'homepage_footer', + formats: [[300, 250]] as [number, number][], + gam_unit_path: '/a/b/footer', + div_id: 'div-footer', + targeting: {}, + }); + + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + await import('../../../src/integrations/gpt/index'); + fetchStub.mockResolvedValue({ ok: true, text: () => Promise.resolve('') } as Response); + + // Source iframe lives under slot A (div-header). + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const fakePort = { postMessage: (s: string) => portMessages.push(s) }; + + window.dispatchEvent( + new MessageEvent('message', { + // adId belongs to slot B (homepage_footer), not slot A's iframe. + data: JSON.stringify({ message: 'Prebid Request', adId: 'footer-uuid' }), + ports: [fakePort as MessagePort], + source, + }) + ); + + await new Promise((r) => setTimeout(r, 50)); + expect(fetchStub).not.toHaveBeenCalled(); + expect(portMessages).toHaveLength(0); + expect(beaconSpy).not.toHaveBeenCalled(); + document.getElementById('div-footer')?.remove(); + }); + it('ignores non-Prebid messages', async () => { await import('../../../src/integrations/gpt/index'); window.dispatchEvent( diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index 5e28f2a25..2ca650e18 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -864,6 +864,64 @@ describe('prebid/installRefreshHandler', () => { ); }); + it('includes configured client-side bidders in refresh ad units', () => { + (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + // Original publisher ad unit carries a client-side rubicon bid. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [ + { bidder: 'trustedServer', params: {} }, + { bidder: 'rubicon', params: { accountId: 1, siteId: 2, zoneId: 3 } }, + ], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { bidder: 'trustedServer', params: { zone: 'homepage' } }, + { bidder: 'rubicon', params: { accountId: 1, siteId: 2, zoneId: 3 } }, + ], + }), + ], + }) + ); + + delete (window as any).__tsjs_prebid; + mockPbjs.adUnits = []; + }); + it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { const originalRefresh = vi.fn(); const clearTargeting = vi.fn(); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 74c2dfdd1..ecd186668 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -95,7 +95,17 @@ ts.servicesEnabled = true; } if (slotsToRefresh.length > 0) { - googletag.pubads().refresh(slotsToRefresh); + // One-shot bypass: this internal refresh delivers the just-applied + // server-side targeting to GAM. If slim-Prebid has already wrapped + // refresh(), it must pass this call straight through — not clear the + // targeting and run a duplicate client-side auction. Mirrors the + // bundle's adInit() in crates/js/lib/src/integrations/gpt/index.ts. + ts.adInitRefreshInProgress = true; + try { + googletag.pubads().refresh(slotsToRefresh); + } finally { + ts.adInitRefreshInProgress = false; + } } }); }; From ceeae6fd89ece5cae3d40b88c4cb829b249aa597 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 15 Jun 2026 13:33:56 +0530 Subject: [PATCH 097/395] Address server-side ad review comments --- crates/js/lib/src/integrations/gpt/index.ts | 43 ++++++- .../lib/test/integrations/gpt/ad_init.test.ts | 121 ++++++++++++++++++ crates/trusted-server-core/build.rs | 6 + .../src/creative_opportunities.rs | 72 +++++++++++ .../src/integrations/gpt.rs | 26 ++++ .../src/integrations/gpt_bootstrap.js | 21 ++- crates/trusted-server-core/src/settings.rs | 119 +++++++++++++++-- 7 files changed, 385 insertions(+), 23 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 8d138a9bf..9fd9ca5c3 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -313,13 +313,46 @@ function injectAdmIntoSlot(divId: string, adm: string): void { function fireWinBillingBeacons(slotId: string, bid: AuctionBidData): void { if (!slotId || (!bid.nurl && !bid.burl)) return; - const beaconKey = `${slotId}|${bid.hb_adid ?? bid.nurl ?? bid.burl ?? ''}`; const fired = (window.tsjs!.firedBeacons ??= {}); - if (fired[beaconKey]) return; + const bidIdentity = bid.hb_adid ?? bid.nurl ?? bid.burl ?? ''; + const urls = [ + ['nurl', bid.nurl], + ['burl', bid.burl], + ] as const; - fired[beaconKey] = true; - if (bid.nurl) navigator.sendBeacon(bid.nurl); - if (bid.burl) navigator.sendBeacon(bid.burl); + for (const [kind, url] of urls) { + if (!url) continue; + + const beaconKey = `${slotId}|${bidIdentity}|${kind}|${url}`; + if (fired[beaconKey]) continue; + + if (queueWinBillingBeacon(url)) { + fired[beaconKey] = true; + } + } +} + +function queueWinBillingBeacon(url: string): boolean { + if (typeof navigator !== 'undefined' && typeof navigator.sendBeacon === 'function') { + try { + if (navigator.sendBeacon(url)) { + return true; + } + } catch (err) { + log.warn('[tsjs-gpt] win/billing sendBeacon failed', err); + } + } + + if (typeof fetch === 'function') { + try { + void fetch(url, { method: 'POST', keepalive: true, mode: 'no-cors' }); + return true; + } catch (err) { + log.warn('[tsjs-gpt] win/billing fetch fallback failed', err); + } + } + + return false; } // ------------------------------------------------------------------ diff --git a/crates/js/lib/test/integrations/gpt/ad_init.test.ts b/crates/js/lib/test/integrations/gpt/ad_init.test.ts index 73d3be9c3..f7bb53e9d 100644 --- a/crates/js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/js/lib/test/integrations/gpt/ad_init.test.ts @@ -707,6 +707,13 @@ describe('installTsRenderBridge', () => { fetchStub = vi.fn(); vi.stubGlobal('fetch', fetchStub); + if (typeof navigator.sendBeacon !== 'function') { + Object.defineProperty(navigator, 'sendBeacon', { + value: vi.fn().mockReturnValue(true), + writable: true, + configurable: true, + }); + } (window as TestWindow).tsjs = { bids: { @@ -747,6 +754,25 @@ describe('installTsRenderBridge', () => { return iframe.contentWindow!; } + async function captureBridgeListener(): Promise<(e: MessageEvent) => unknown> { + let bridgeListener: ((e: MessageEvent) => unknown) | undefined; + const origAdd = window.addEventListener.bind(window); + const addSpy = vi + .spyOn(window, 'addEventListener') + .mockImplementation( + (type: string, handler: EventListenerOrEventListenerObject, opts?: unknown) => { + if (type === 'message') bridgeListener = handler as (e: MessageEvent) => unknown; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + origAdd(type, handler as EventListener, opts as any); + } + ); + await import('../../../src/integrations/gpt/index'); + addSpy.mockRestore(); + + expect(bridgeListener, 'bridge listener should be registered').toBeDefined(); + return bridgeListener!; + } + it('calls stopImmediatePropagation and fetches PBS Cache for a TS bid', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const mockAd = '
Test Creative
'; @@ -892,6 +918,101 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('falls back to keepalive fetch when sendBeacon is unavailable', async () => { + const originalSendBeacon = navigator.sendBeacon; + Object.defineProperty(navigator, 'sendBeacon', { + value: undefined, + writable: true, + configurable: true, + }); + + try { + (window as TestWindow).tsjs.bids.homepage_header = { + hb_adid: 'debug-no-beacon', + hb_bidder: 'mocktioneer', + hb_pb: '0.20', + nurl: 'https://debug.example/win', + burl: 'https://debug.example/bill', + adm: '
Debug Creative
', + }; + + const bridgeListener = await captureBridgeListener(); + const portMessages: string[] = []; + const fakePort = { postMessage: (s: string) => portMessages.push(s) }; + const source = createTrustedSlotIframe(); + + expect(() => + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'debug-no-beacon' }), + ports: [fakePort], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ) + ).not.toThrow(); + + expect(fetchStub).toHaveBeenCalledWith('https://debug.example/win', { + method: 'POST', + keepalive: true, + mode: 'no-cors', + }); + expect(fetchStub).toHaveBeenCalledWith('https://debug.example/bill', { + method: 'POST', + keepalive: true, + mode: 'no-cors', + }); + } finally { + Object.defineProperty(navigator, 'sendBeacon', { + value: originalSendBeacon, + writable: true, + configurable: true, + }); + } + }); + + it('falls back to keepalive fetch when sendBeacon rejects the payload', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(false); + (window as TestWindow).tsjs.bids.homepage_header = { + hb_adid: 'debug-rejected-beacon', + hb_bidder: 'mocktioneer', + hb_pb: '0.20', + nurl: 'https://debug.example/win', + burl: 'https://debug.example/bill', + adm: '
Debug Creative
', + }; + + const bridgeListener = await captureBridgeListener(); + const portMessages: string[] = []; + const fakePort = { postMessage: (s: string) => portMessages.push(s) }; + const source = createTrustedSlotIframe(); + const event = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'debug-rejected-beacon' }), + ports: [fakePort], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(event); + + expect(beaconSpy).toHaveBeenCalledWith('https://debug.example/win'); + expect(beaconSpy).toHaveBeenCalledWith('https://debug.example/bill'); + expect(fetchStub).toHaveBeenCalledWith('https://debug.example/win', { + method: 'POST', + keepalive: true, + mode: 'no-cors', + }); + expect(fetchStub).toHaveBeenCalledWith('https://debug.example/bill', { + method: 'POST', + keepalive: true, + mode: 'no-cors', + }); + + bridgeListener(event); + expect(fetchStub).toHaveBeenCalledTimes(2); + beaconSpy.mockRestore(); + }); + it('ignores message when adId does not match any TS bid', async () => { await import('../../../src/integrations/gpt/index'); fetchStub.mockResolvedValue({ ok: true, text: () => Promise.resolve('') } as Response); diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index a4fe174ce..1787d5063 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -76,6 +76,12 @@ mod creative_opportunities { impl CreativeOpportunitiesConfig { /// No-op stub — pattern compilation only runs at runtime. pub fn compile_slots(&mut self) {} + + /// No-op stub — full slot-shape validation runs at runtime against + /// the real creative opportunity types. + pub fn validate_runtime(&self) -> Result<(), String> { + Ok(()) + } } /// Stub — the typed `slot` vec is always empty in the build context (see diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 645ba423f..67728bd28 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -51,6 +51,20 @@ impl CreativeOpportunitiesConfig { slot.compile_patterns(); } } + + /// Validate all slot definitions after runtime preparation. + /// + /// # Errors + /// + /// Returns an error string when a slot has an invalid identifier, page + /// pattern set, format list, dimensions, or resolved GAM unit path. + pub fn validate_runtime(&self) -> Result<(), String> { + for slot in &self.slot { + slot.validate_runtime(&self.gam_network_id)?; + } + + Ok(()) + } } /// A single ad placement opportunity on the publisher's site. @@ -94,6 +108,54 @@ pub struct CreativeOpportunitySlot { } impl CreativeOpportunitySlot { + /// Validate the slot shape after [`compile_patterns`](Self::compile_patterns) has run. + /// + /// # Errors + /// + /// Returns an error string when required slot fields are empty, invalid, + /// or semantically unusable at runtime. + pub fn validate_runtime(&self, gam_network_id: &str) -> Result<(), String> { + validate_slot_id(&self.id)?; + + if self.page_patterns.is_empty() { + return Err(format!( + "slot `{}` must include at least one page pattern", + self.id + )); + } + + if self.compiled_patterns.is_empty() { + return Err(format!( + "slot `{}` must include at least one valid page pattern", + self.id + )); + } + + if self.formats.is_empty() { + return Err(format!( + "slot `{}` must include at least one format", + self.id + )); + } + + for format in &self.formats { + format.validate_runtime(&self.id)?; + } + + if self + .resolved_gam_unit_path(gam_network_id) + .trim() + .is_empty() + { + return Err(format!( + "slot `{}` resolved GAM unit path must not be empty", + self.id + )); + } + + Ok(()) + } + /// Returns `true` if `path` matches any of this slot's [`page_patterns`](Self::page_patterns). /// /// Patterns use glob syntax (e.g., `"/20**"` matches any path starting with `/20`, @@ -236,6 +298,16 @@ pub struct CreativeOpportunityFormat { } impl CreativeOpportunityFormat { + fn validate_runtime(&self, slot_id: &str) -> Result<(), String> { + if self.width == 0 || self.height == 0 { + return Err(format!( + "slot `{slot_id}` format must have positive width and height" + )); + } + + Ok(()) + } + fn to_ad_format(&self) -> AdFormat { AdFormat { media_type: self.media_type.clone(), diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index dedb830f9..0a847651d 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -1118,6 +1118,32 @@ mod tests { ); } + #[test] + fn head_inserts_bootstrap_uses_css_safe_div_prefix_lookup() { + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let combined = integration.head_inserts(&ctx).join(""); + assert!( + combined.contains("querySelectorAll(\"[id]\")"), + "bootstrap should scan ID-bearing elements instead of interpolating div_id into CSS" + ); + assert!( + combined.contains(".startsWith(slot.div_id)"), + "bootstrap should match metacharacter-containing div_id prefixes with startsWith" + ); + assert!( + !combined.contains("[id^='\" + slot.div_id"), + "bootstrap must not build a CSS attribute selector from raw div_id" + ); + } + #[test] fn head_inserts_bootstrap_guards_enable_services_with_idempotency_flag() { let config = test_config(); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index ecd186668..90eb2181b 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -31,14 +31,23 @@ // All slots to refresh (TS-defined + publisher-owned reused). var slotsToRefresh = []; slots.forEach(function (slot) { - // Resolve actual div ID: exact match first, then prefix query. + // Resolve actual div ID: exact match first, then safe prefix scan. // div_id in config may be a stable prefix (e.g. "ad-header-0-") when // the suffix is dynamically generated by the framework at render time. - var el = - document.getElementById(slot.div_id) || - document.querySelector( - "[id^='" + slot.div_id + "']:not([id$='-container'])", - ); + var el = document.getElementById(slot.div_id); + if (!el) { + var idElements = document.querySelectorAll("[id]"); + for (var i = 0; i < idElements.length; i++) { + var candidate = idElements[i]; + if ( + candidate.id.startsWith(slot.div_id) && + !candidate.id.endsWith("-container") + ) { + el = candidate; + break; + } + } + } if (!el) return; var actualDivId = el.id; var b = bids[slot.id] || {}; diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 286ab4234..24c933552 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1786,7 +1786,7 @@ impl Settings { /// /// Returns a configuration error if any cached runtime artifact cannot be /// prepared, if any handler path regex does not compile, or if a creative - /// opportunity slot has an invalid ID. + /// opportunity slot is invalid. pub fn prepare_runtime(&mut self) -> Result<(), Report> { self.image_optimizer.prepare_runtime()?; self.proxy.prepare_runtime()?; @@ -1798,16 +1798,14 @@ impl Settings { if let Some(co) = &mut self.creative_opportunities { co.compile_slots(); - // Slot IDs flow into injected HTML/JS and provider payloads, and - // can arrive via TRUSTED_SERVER__ env overrides that bypass any - // static config review — validate them on every load path. - for slot in &co.slot { - crate::creative_opportunities::validate_slot_id(&slot.id).map_err(|err| { - Report::new(TrustedServerError::Configuration { - message: format!("Invalid creative opportunity slot id: {err}"), - }) - })?; - } + // Slots flow into injected HTML/JS, provider payloads, and GPT + // calls. Env/private config can bypass static review, so validate + // the full runtime shape on every load path. + co.validate_runtime().map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!("Invalid creative opportunity slot config: {err}"), + }) + })?; } Ok(()) @@ -4681,11 +4679,108 @@ formats = [{ width = 300, height = 250 }] "#; let err = Settings::from_toml(toml).expect_err("should reject invalid slot id"); assert!( - format!("{err:?}").contains("Invalid creative opportunity slot id"), + format!("{err:?}").contains("Invalid creative opportunity slot config"), "error should mention the invalid slot id, got: {err:?}" ); } + fn creative_opportunity_settings_toml(slot_body: &str) -> String { + format!( + r#" +[[handlers]] +path = "^/_ts/admin" +username = "admin" +password = "unit-test-admin-secret" + +[publisher] +domain = "example.com" +cookie_domain = ".example.com" +origin_url = "https://origin.example.com" +proxy_secret = "secret" + +[ec] +passphrase = "test-secret-key-32-bytes-minimum" + +[creative_opportunities] +gam_network_id = "21765378893" + +[[creative_opportunities.slot]] +{slot_body} +"# + ) + } + + fn assert_creative_opportunity_slot_config_rejected(slot_body: &str, expected: &str) { + let toml = creative_opportunity_settings_toml(slot_body); + let err = Settings::from_toml(&toml) + .expect_err("should reject malformed creative opportunity slot"); + assert!( + format!("{err:?}").contains(expected), + "error should contain {expected:?}, got: {err:?}" + ); + } + + #[test] + fn settings_rejects_creative_opportunity_slot_without_page_patterns() { + assert_creative_opportunity_slot_config_rejected( + r#" +id = "atf" +page_patterns = [] +formats = [{ width = 300, height = 250 }] +"#, + "must include at least one page pattern", + ); + } + + #[test] + fn settings_rejects_creative_opportunity_slot_without_valid_page_patterns() { + assert_creative_opportunity_slot_config_rejected( + r#" +id = "atf" +page_patterns = ["["] +formats = [{ width = 300, height = 250 }] +"#, + "must include at least one valid page pattern", + ); + } + + #[test] + fn settings_rejects_creative_opportunity_slot_without_formats() { + assert_creative_opportunity_slot_config_rejected( + r#" +id = "atf" +page_patterns = ["/"] +formats = [] +"#, + "must include at least one format", + ); + } + + #[test] + fn settings_rejects_creative_opportunity_slot_with_zero_dimensions() { + assert_creative_opportunity_slot_config_rejected( + r#" +id = "atf" +page_patterns = ["/"] +formats = [{ width = 0, height = 250 }] +"#, + "must have positive width and height", + ); + } + + #[test] + fn settings_rejects_creative_opportunity_slot_with_empty_gam_unit_path() { + assert_creative_opportunity_slot_config_rejected( + r#" +id = "atf" +gam_unit_path = "" +page_patterns = ["/"] +formats = [{ width = 300, height = 250 }] +"#, + "resolved GAM unit path must not be empty", + ); + } + #[test] fn admin_endpoints_match_fastly_router() { let router_source = include_str!("../../trusted-server-adapter-fastly/src/main.rs"); From 911a6456b44cf9d6b62f39c2cd067d21c7f70415 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 15 Jun 2026 20:43:26 +0530 Subject: [PATCH 098/395] Restore publisher platform-http-client test on the merged signature Re-add publisher_request_uses_platform_http_client_with_http_types, dropped during the main merge because it called the pre-feature 4-arg handle_publisher_request. A run_publisher_proxy test helper supplies the no-auction EC/AuctionDispatch wiring so the test body stays a plain (settings, registry, services, req) proxy call. --- crates/trusted-server-core/src/publisher.rs | 73 ++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1e75197d6..3c89b923d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1931,7 +1931,9 @@ mod tests { use super::*; use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::integrations::IntegrationRegistry; - use crate::platform::test_support::noop_services; + use crate::platform::test_support::{ + build_services_with_http_client, noop_services, StubHttpClient, + }; use crate::test_support::tests::create_test_settings; use edgezero_core::body::Body as EdgeBody; use http::{header, Method, Request as HttpRequest, StatusCode}; @@ -2162,6 +2164,75 @@ mod tests { ); } + /// Drive `handle_publisher_request` with no creative opportunities — a plain + /// proxy with no server-side auction. Hides the auction/EC wiring so callers + /// read like a simple `(settings, registry, services, req)` proxy. + async fn run_publisher_proxy( + settings: &Settings, + integration_registry: &IntegrationRegistry, + services: &RuntimeServices, + req: Request, + ) -> PublisherResponse { + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let fastly_req = crate::compat::to_fastly_request_ref(&req); + let mut ec_context = + EcContext::read_from_request(settings, &fastly_req).expect("should read EC context"); + handle_publisher_request( + settings, + integration_registry, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + ) + .await + .expect("should proxy publisher request") + } + + #[tokio::test] + async fn publisher_request_uses_platform_http_client_with_http_types() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"origin response".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = match run_publisher_proxy(&settings, ®istry, &services, req).await { + PublisherResponse::Buffered(r) => r, + PublisherResponse::PassThrough { mut response, body } => { + *response.body_mut() = body; + response + } + PublisherResponse::Stream { response, .. } => response, + }; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + String::from_utf8(response.into_body().into_bytes().to_vec()) + .expect("response body should be valid UTF-8"), + "origin response" + ); + assert_eq!( + stub.recorded_backend_names(), + vec!["stub-backend".to_string()], + "should proxy through the platform http client" + ); + } + #[test] fn test_content_type_detection() { let test_cases = vec![ From 89281f0edfc24475b3e8978e69c2526522442733 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 15 Jun 2026 21:51:47 +0530 Subject: [PATCH 099/395] Gate POST /auction behind the server-side auction consent check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The publisher-navigation and /__ts/page-bids paths fail closed for GDPR or unknown jurisdictions that lack effective TCF Purpose 1, but POST /auction proceeded straight to run_auction after only stripping EC IDs/EIDs — still dispatching PBS/APS calls and forwarding request-derived signals (UA/IP/geo, and cookies under some Prebid consent-forwarding modes) for traffic the gate says must not run a server-side auction. Apply consent_allows_server_side_auction before resolving EIDs or contacting providers; when it denies, return an empty no-bid OpenRTB response without invoking run_auction. Add a regression test that registers a panic-on-bid provider and proves a GDPR/unknown request lacking Purpose 1 returns no bids without contacting any provider. Route the orchestration-failure /auction tests through a non-regulated geo so they still exercise the provider path. --- .../src/route_tests.rs | 28 +++- .../src/auction/endpoints.rs | 150 +++++++++++++++++- 2 files changed, 176 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/route_tests.rs b/crates/trusted-server-adapter-fastly/src/route_tests.rs index 7e2aa23f7..c616223bc 100644 --- a/crates/trusted-server-adapter-fastly/src/route_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/route_tests.rs @@ -378,6 +378,23 @@ fn us_california_geo() -> GeoInfo { } } +/// Geo resolving to a non-regulated jurisdiction, so the server-side auction +/// consent gate (which fails closed for GDPR/unknown jurisdictions without TCF +/// Purpose 1) allows the auction to proceed. Used by `/auction` route tests +/// that exercise orchestration behavior rather than consent. +fn non_regulated_geo() -> GeoInfo { + GeoInfo { + city: "Example City".to_string(), + country: "AU".to_string(), + continent: "OC".to_string(), + latitude: -33.8, + longitude: 151.2, + metro_code: 0, + region: Some("NSW".to_string()), + asn: None, + } +} + fn valid_ec_id() -> String { format!("{}.Abc123", "a".repeat(64)) } @@ -594,7 +611,16 @@ fn route_auction_with_stack( let req = Request::post("https://test.com/auction") .with_header(header::CONTENT_TYPE, "application/json") .with_body(body.into()); - let services = test_runtime_services(&req); + // Resolve to a non-regulated jurisdiction so the server-side auction consent + // gate allows the auction; these tests assert orchestration behavior, not + // consent gating (covered separately in endpoints.rs). + let services = test_runtime_services_with_secret_http_client_and_geo( + &req, + Arc::new(NoopBackend), + Arc::new(NoopSecretStore), + Arc::new(NoopHttpClient) as Arc, + Arc::new(FixedGeo(non_regulated_geo())), + ); let route_result = futures::executor::block_on(route_request( settings, diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index f72954212..5ed59aae5 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -1,12 +1,15 @@ //! HTTP endpoint handlers for auction requests. +use std::collections::HashMap; + use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{header, Request, Response, StatusCode}; use serde_json::Value as JsonValue; use crate::auction::formats::AdRequest; -use crate::consent::gate_eids_by_consent; +use crate::auction::orchestrator::OrchestrationResult; +use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::COOKIE_TS_EIDS; use crate::ec::eids::{resolve_partner_ids, to_eids}; use crate::ec::kv::KvIdentityGraph; @@ -163,6 +166,43 @@ pub async fn handle_auction( }; let consent_context = ec_context.consent().clone(); + // Server-side auction consent gate. The publisher-navigation and + // `/__ts/page-bids` paths fail closed for GDPR/unknown jurisdictions that + // lack effective TCF Purpose 1. `/auction` is the programmatic entry point + // for the same server-side auction, so it must gate identically: returning + // a no-bid response here prevents outbound PBS/APS calls and the forwarding + // of request-derived signals (UA/IP/geo, and cookies under some Prebid + // consent-forwarding modes) for traffic that must not run an auction. + if !consent_allows_server_side_auction(&consent_context) { + log::info!( + "/auction: server-side auction consent gate denied; returning no-bid response without contacting providers" + ); + // Build the request shape locally (no outbound calls, no geo lookup, no + // EID resolution) so the no-bid OpenRTB response echoes the request id. + let auction_request = convert_tsjs_to_auction_request( + &body, + settings, + services, + &http_req, + consent_context, + ec_id, + None, + )?; + let empty_result = OrchestrationResult { + provider_responses: Vec::new(), + mediator_response: None, + winning_bids: HashMap::new(), + total_time_ms: 0, + metadata: HashMap::new(), + }; + return convert_to_openrtb_response( + &empty_result, + settings, + &auction_request, + ec_context.ec_allowed(), + ); + } + // Parse client-provided EIDs from the current request body. When the // current request does not include them, fall back to the persisted // `ts-eids` cookie so later requests can still forward the browser's @@ -444,12 +484,19 @@ pub(crate) fn merge_auction_eids( #[cfg(test)] mod tests { use super::*; + use crate::auction::config::AuctionConfig; + use crate::auction::provider::AuctionProvider; + use crate::auction::types::{AuctionRequest, AuctionResponse}; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::ConsentContext; use crate::openrtb::Uid; + use crate::platform::test_support::noop_services; + use crate::platform::{PlatformPendingRequest, PlatformResponse}; + use crate::test_support::tests::create_test_settings; use base64::engine::general_purpose::STANDARD as BASE64; use base64::Engine as _; use serde_json::json; + use std::sync::Arc; fn make_ec_context(jurisdiction: Jurisdiction, ec_value: Option<&str>) -> EcContext { EcContext::new_for_test( @@ -461,6 +508,107 @@ mod tests { ) } + /// Provider that fails the test if it is ever contacted. Used to prove the + /// `/auction` consent gate short-circuits before any outbound bid request. + struct PanicOnBidProvider; + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for PanicOnBidProvider { + fn provider_name(&self) -> &'static str { + "panic_provider" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + panic!("provider must not be contacted when the consent gate denies the auction"); + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("provider must not parse a response when the auction is gated off"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _timeout_ms: u32) -> Option { + Some("panic-backend".to_string()) + } + } + + #[tokio::test] + async fn auction_endpoint_consent_gate_returns_no_bid_without_contacting_providers() { + // GDPR/unknown jurisdiction lacking effective TCF Purpose 1 must not run + // a server-side auction. The /auction endpoint must short-circuit to a + // no-bid response before dispatching to any provider — matching the + // publisher-navigation and /__ts/page-bids paths. + let settings = create_test_settings(); + let config = AuctionConfig { + enabled: true, + providers: vec!["panic_provider".to_string()], + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(PanicOnBidProvider)); + let services = noop_services(); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); + + let body = json!({ + "adUnits": [ + { + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + } + ] + }); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize body"), + )) + .expect("should build auction request"); + + let response = handle_auction( + &settings, + &orchestrator, + None, + None, + &ec_context, + &services, + req, + ) + .await + .expect("gated auction should still return a valid response"); + + assert_eq!( + response.status(), + StatusCode::OK, + "gated auction should return a 200 no-bid response" + ); + let body_bytes = response.into_body().into_bytes(); + let parsed: JsonValue = + serde_json::from_slice(&body_bytes).expect("response body should be valid JSON"); + let seatbid_empty = match parsed.get("seatbid").and_then(JsonValue::as_array) { + Some(seatbid) => seatbid.is_empty(), + None => true, + }; + assert!( + seatbid_empty, + "gated auction must return no bids, got: {parsed}" + ); + } + #[test] fn resolve_auction_eids_returns_none_without_kv() { let registry = PartnerRegistry::empty(); From 4d4fb1b238902b74ede109d14166ca0ba2cc430e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 15 Jun 2026 21:51:47 +0530 Subject: [PATCH 100/395] Validate creative-opportunity slots at build time build.rs deserialized slots into a stub whose validate_runtime was a no-op and only checked slot-id syntax, so an invalid trusted-server.toml or TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT override (empty page_patterns, empty formats, zero dimensions, empty resolved GAM unit path) passed CI and got embedded, then failed at request time as a configuration error. Extract the validation into creative_slot_build_check, shared by build.rs (via #[path]) and the crate test build (via #[cfg(test)] mod) so the rules run under cargo test. It mirrors CreativeOpportunitySlot::validate_runtime and runs against the merged config (base TOML plus TRUSTED_SERVER__* env overrides) before the config is serialized and embedded, so an invalid slot fails the build and is never persisted. --- crates/trusted-server-core/build.rs | 53 ++--- .../src/creative_slot_build_check.rs | 201 ++++++++++++++++++ crates/trusted-server-core/src/lib.rs | 4 + 3 files changed, 232 insertions(+), 26 deletions(-) create mode 100644 crates/trusted-server-core/src/creative_slot_build_check.rs diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index 1787d5063..cee32e259 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -100,6 +100,10 @@ mod creative_opportunities { #[path = "src/settings.rs"] mod settings; +#[path = "src/creative_slot_build_check.rs"] +mod creative_slot_build_check; + +use creative_slot_build_check::validate_creative_slot; use std::fs; use std::path::Path; @@ -118,38 +122,24 @@ fn main() { let toml_content = fs::read_to_string(init_config_path) .unwrap_or_else(|_| panic!("Failed to read {init_config_path:?}")); - // Merge base TOML with environment variable overrides and write output. + // Merge base TOML with environment variable overrides. // Panics if admin endpoints are not covered by a handler. let settings = settings::Settings::from_toml_and_env(&toml_content) .expect("Failed to parse settings at build time"); - let merged_toml = - toml::to_string_pretty(&settings).expect("Failed to serialize settings to TOML"); - - // Only write when content changes to avoid unnecessary recompilation. - let dest_path = Path::new(TRUSTED_SERVER_OUTPUT_CONFIG_PATH); - let current = fs::read_to_string(dest_path).unwrap_or_default(); - if current != merged_toml { - fs::write(dest_path, merged_toml) - .unwrap_or_else(|_| panic!("Failed to write {dest_path:?}")); - } - - // Validate slot IDs from [creative_opportunities.slot] in trusted-server.toml - let slot_id_re = regex::Regex::new(r"^[A-Za-z0-9_\-]+$").expect("should compile regex"); + // Validate [creative_opportunities.slot] entries from the *merged* config + // (base trusted-server.toml plus any TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT + // env overrides) before it is serialized and embedded. This mirrors the + // runtime validator (CreativeOpportunitySlot::validate_runtime) — the build + // context uses a stub whose validate_runtime is a no-op, so without this an + // invalid slot would pass CI and surface as a request-time configuration + // error / service outage. The validator is shared with the crate (see + // `creative_slot_build_check`) so it stays under test. Running it before the + // write also means a rejected config is never persisted to the embedded file. if let Some(co) = &settings.creative_opportunities { for slot in &co.slot_raw { - if let Some(id) = slot.get("id").and_then(|v| v.as_str()) { - if !slot_id_re.is_match(id) { - panic!( - "trusted-server.toml [creative_opportunities.slot]: slot id '{}' is invalid; \ - only [A-Za-z0-9_-] allowed", - id - ); - } - } else { - panic!( - "trusted-server.toml [creative_opportunities.slot]: a slot entry is missing the required 'id' field" - ); + if let Err(err) = validate_creative_slot(slot, &co.gam_network_id) { + panic!("trusted-server.toml [creative_opportunities.slot]: {err}"); } } if !co.slot_raw.is_empty() { @@ -159,4 +149,15 @@ fn main() { ); } } + + let merged_toml = + toml::to_string_pretty(&settings).expect("Failed to serialize settings to TOML"); + + // Only write when content changes to avoid unnecessary recompilation. + let dest_path = Path::new(TRUSTED_SERVER_OUTPUT_CONFIG_PATH); + let current = fs::read_to_string(dest_path).unwrap_or_default(); + if current != merged_toml { + fs::write(dest_path, merged_toml) + .unwrap_or_else(|_| panic!("Failed to write {dest_path:?}")); + } } diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs new file mode 100644 index 000000000..55d17f918 --- /dev/null +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -0,0 +1,201 @@ +//! Build-time validation for creative-opportunity slot definitions. +//! +//! This module is compiled in two contexts: +//! - by `build.rs` (via `#[path]`), which runs it against the raw slot JSON +//! merged from `trusted-server.toml` and `TRUSTED_SERVER__*` env overrides +//! before the config is embedded into the binary; +//! - by the crate's test build (via `#[cfg(test)] mod`), so the rules below are +//! exercised under `cargo test`. +//! +//! It mirrors the runtime validator +//! (`CreativeOpportunitySlot::validate_runtime`) so an invalid slot fails the +//! build instead of surfacing as a request-time configuration error. It reads +//! raw JSON (not the typed runtime struct) because the typed slot vec is +//! intentionally empty in the build context, keeping `build.rs` free of the +//! full runtime dependency graph. + +/// Returns `true` when `id` is non-empty and only `[A-Za-z0-9_-]`. +fn is_valid_slot_id(id: &str) -> bool { + !id.is_empty() + && id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') +} + +/// Validate a single raw creative-opportunity slot. +/// +/// Mirrors the runtime checks in `CreativeOpportunitySlot::validate_runtime`: +/// a syntactically safe non-empty id, at least one non-empty page pattern, at +/// least one format with positive dimensions, and a non-empty resolved GAM unit +/// path. Returns an error string describing the first problem found. +/// +/// # Errors +/// +/// Returns an error string when the slot is missing required fields, has an +/// invalid id, has no usable page pattern or format, has a zero-dimension +/// format, or resolves to an empty GAM unit path. +pub(crate) fn validate_creative_slot( + slot: &serde_json::Value, + gam_network_id: &str, +) -> Result<(), String> { + let id = match slot.get("id").and_then(serde_json::Value::as_str) { + Some(id) => id, + None => return Err("a slot entry is missing the required 'id' field".to_string()), + }; + if id.is_empty() { + return Err("slot id must not be empty".to_string()); + } + if !is_valid_slot_id(id) { + return Err(format!( + "slot id '{id}' is invalid; only [A-Za-z0-9_-] allowed" + )); + } + + // At least one non-empty page pattern. + let has_valid_pattern = slot + .get("page_patterns") + .and_then(serde_json::Value::as_array) + .is_some_and(|patterns| { + patterns + .iter() + .any(|p| p.as_str().is_some_and(|s| !s.trim().is_empty())) + }); + if !has_valid_pattern { + return Err(format!( + "slot `{id}` must include at least one non-empty page pattern" + )); + } + + // At least one format, each with positive width and height. + match slot.get("formats").and_then(serde_json::Value::as_array) { + Some(formats) if !formats.is_empty() => { + for format in formats { + let width = format.get("width").and_then(serde_json::Value::as_u64); + let height = format.get("height").and_then(serde_json::Value::as_u64); + if !matches!((width, height), (Some(w), Some(h)) if w > 0 && h > 0) { + return Err(format!( + "slot `{id}` format must have positive width and height" + )); + } + } + } + _ => { + return Err(format!("slot `{id}` must include at least one format")); + } + } + + // Resolved GAM unit path must not be empty. An explicit override is used + // when present; otherwise it is derived as `//`. + let resolved_gam_unit_path = match slot + .get("gam_unit_path") + .and_then(serde_json::Value::as_str) + { + Some(path) => path.to_string(), + None => format!("/{gam_network_id}/{id}"), + }; + if resolved_gam_unit_path.trim().is_empty() { + return Err(format!( + "slot `{id}` resolved GAM unit path must not be empty" + )); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::validate_creative_slot; + use serde_json::json; + + #[test] + fn accepts_a_well_formed_slot() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }] + }); + assert!(validate_creative_slot(&slot, "123456789").is_ok()); + } + + #[test] + fn accepts_explicit_gam_unit_path_override() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/"], + "formats": [{ "width": 300, "height": 250 }], + "gam_unit_path": "/123456789/publisher/atf" + }); + assert!(validate_creative_slot(&slot, "123456789").is_ok()); + } + + #[test] + fn rejects_empty_formats() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("empty formats must fail at build time"); + assert!(err.contains("at least one format"), "got: {err}"); + } + + #[test] + fn rejects_zero_dimension_format() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 0, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("zero dimensions must fail at build time"); + assert!(err.contains("positive width and height"), "got: {err}"); + } + + #[test] + fn rejects_empty_page_patterns() { + let slot = json!({ + "id": "atf", + "page_patterns": [], + "formats": [{ "width": 300, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("empty page patterns must fail at build time"); + assert!(err.contains("page pattern"), "got: {err}"); + } + + #[test] + fn rejects_blank_page_pattern_strings() { + let slot = json!({ + "id": "atf", + "page_patterns": [" "], + "formats": [{ "width": 300, "height": 250 }] + }); + assert!(validate_creative_slot(&slot, "123456789").is_err()); + } + + #[test] + fn rejects_blank_gam_unit_path_override() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "gam_unit_path": " " + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("blank GAM unit path must fail at build time"); + assert!(err.contains("GAM unit path"), "got: {err}"); + } + + #[test] + fn rejects_missing_id() { + let slot = json!({ "page_patterns": ["/"], "formats": [{ "width": 1, "height": 1 }] }); + assert!(validate_creative_slot(&slot, "net").is_err()); + } + + #[test] + fn rejects_invalid_id_characters() { + let slot = json!({ "id": "a b", "page_patterns": ["/"], "formats": [{ "width": 1, "height": 1 }] }); + assert!(validate_creative_slot(&slot, "net").is_err()); + } +} diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index de71e010c..23ea63045 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -42,6 +42,10 @@ pub mod constants; pub mod cookies; pub mod creative; pub mod creative_opportunities; +// Build-time slot validation, shared with `build.rs` via `#[path]`. Compiled +// here only under test so its rules stay exercised by `cargo test`. +#[cfg(test)] +mod creative_slot_build_check; pub mod ec; pub(crate) mod edge_cookie; pub mod error; From c99bac8b8763c20f7e7045285bae2e30df494db8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 09:26:48 +0530 Subject: [PATCH 101/395] Restore mediated render/accounting fields on the synchronous auction path run_parallel_mediation parsed the mediator response through parse_response, which (for adserver_mock) drops nurl/burl/ad_id and PBS cache fields restored only in parse_response_with_context. The synchronous mediation path used by POST /auction and /__ts/page-bids could therefore return mediated cache bids without hb_adid / cache metadata, breaking creative rendering and win/billing beacons even though the dispatched collect path preserves them. Call parse_response_with_context with the mediator context (which carries the collected SSP responses), matching the dispatched collect path. Add a regression test proving a mediated bid keeps its restored nurl/ad_id through run_auction. --- .../src/auction/orchestrator.rs | 161 +++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index ba06e6c74..dc3bb5e83 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -266,8 +266,14 @@ impl AuctionOrchestrator { })?; let response_time_ms = start_time.elapsed().as_millis() as u64; + // Use the context-aware parse so mediators (e.g. adserver_mock) can + // restore nurl/burl/ad_id and PBS cache fields from the collected SSP + // responses. The dispatched collect path already does this; the + // synchronous mediation path used by POST /auction and + // /__ts/page-bids must match or mediated cache bids lose the metadata + // needed for creative rendering and win/billing beacons. let mediator_resp = mediator - .parse_response(platform_resp, response_time_ms) + .parse_response_with_context(platform_resp, response_time_ms, &mediator_context) .await .change_context(TrustedServerError::Auction { message: format!("Mediator {} parse failed", mediator.provider_name()), @@ -1211,6 +1217,159 @@ mod tests { } } + /// Mediator whose context-aware parse restores `nurl`/`ad_id` (mirroring + /// `adserver_mock`), while its context-free parse does not. Lets a test prove + /// the synchronous mediation path calls `parse_response_with_context`. + struct CacheRestoringMediator; + + fn mediated_bid(nurl: Option) -> Bid { + Bid { + slot_id: "header-banner".to_string(), + price: Some(2.5), + currency: "USD".to_string(), + creative: Some("
ad
".to_string()), + adomain: None, + bidder: "mediator".to_string(), + width: 728, + height: 90, + nurl: nurl.clone(), + burl: nurl, + ad_id: Some("creative-123".to_string()), + cache_id: Some("cache-abc".to_string()), + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + } + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for CacheRestoringMediator { + fn provider_name(&self) -> &'static str { + "mediator" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let req = PlatformHttpRequest::new( + http::Request::builder() + .method("POST") + .uri("https://example.com/mediate") + .body(edgezero_core::body::Body::empty()) + .expect("should build mediator request"), + "mediator-backend", + ); + context + .services + .http_client() + .send_async(req) + .await + .change_context(TrustedServerError::Auction { + message: "mediator launch failed".to_string(), + }) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + // Context-free path: cannot restore SSP-only render/accounting fields. + Ok(AuctionResponse::success( + "mediator", + vec![mediated_bid(None)], + response_time_ms, + )) + } + + async fn parse_response_with_context( + &self, + _response: PlatformResponse, + response_time_ms: u64, + _context: &AuctionContext<'_>, + ) -> Result> { + // Context-aware path: restores nurl/ad_id from the collected SSP bids. + Ok(AuctionResponse::success( + "mediator", + vec![mediated_bid(Some("https://nurl.example/win".to_string()))], + response_time_ms, + )) + } + + fn timeout_ms(&self) -> u32 { + 2000 + } + + fn backend_name(&self, _timeout_ms: u32) -> Option { + Some("mediator-backend".to_string()) + } + } + + #[tokio::test] + async fn mediated_bid_preserves_restored_fields_through_run_auction() { + // run_parallel_mediation must parse the mediator response via + // parse_response_with_context so cache/nurl fields restored from SSP + // responses survive the synchronous mediation path (POST /auction, + // /__ts/page-bids), matching the dispatched collect path. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); // bidder send_async + stub.push_response(200, b"{}".to_vec()); // mediator send_async + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let config = AuctionConfig { + enabled: true, + providers: vec!["bidder".to_string()], + mediator: Some("mediator".to_string()), + timeout_ms: 2000, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(StubAuctionProvider { + name: "bidder", + backend: "bidder-backend", + })); + orchestrator.register_provider(Arc::new(CacheRestoringMediator)); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 2000, + provider_responses: None, + services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("mediated auction should complete"); + + let bid = result + .winning_bids + .get("header-banner") + .expect("mediator should produce a winning bid for the slot"); + assert_eq!( + bid.nurl.as_deref(), + Some("https://nurl.example/win"), + "synchronous mediation must restore nurl via parse_response_with_context" + ); + assert_eq!( + bid.ad_id.as_deref(), + Some("creative-123"), + "mediated bid must keep its restored ad_id" + ); + } + fn create_test_auction_request() -> AuctionRequest { AuctionRequest { id: "test-auction-123".to_string(), From b9d2d06483ca642465c0c892ee1e1a32884f3379 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 09:26:48 +0530 Subject: [PATCH 102/395] Display TS-defined GPT slots instead of only refreshing them In the fallback path where Trusted Server defines a GPT slot itself, the code called defineSlot().addService() then refresh(), but never googletag.display() for the new slot. GPT requires a display() call to register/render a slot, so TS-owned first-impression slots no-op ("defineSlot was called without a matching display call") and miss impressions. Reused publisher-owned slots are unaffected because the publisher already displayed them. Track TS-defined slot element IDs separately, display() them once after services are enabled, and keep refresh() for reused publisher-owned slots only. Mirror the change in the inline gpt_bootstrap.js. Add Vitest coverage for the TS-owned display path and keep the refresh-bypass test on a reused slot. --- crates/js/lib/src/integrations/gpt/index.ts | 32 ++++++++--- .../lib/test/integrations/gpt/ad_init.test.ts | 53 ++++++++++++++++++- .../src/integrations/gpt_bootstrap.js | 23 ++++++-- 3 files changed, 97 insertions(+), 11 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 9fd9ca5c3..2effbc593 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -392,8 +392,14 @@ export function installTsAdInit(): void { // Slots TS defined itself — tracked for SPA destroy. Publisher-owned // slots are reused but never destroyed by TS on navigation. const newSlots: GoogleTagSlot[] = []; - // All slots to refresh (TS-defined + publisher-owned reused). + // Publisher-owned slots TS reused — refreshed to pick up server-side + // targeting. The publisher already display()ed these. const slotsToRefresh: GoogleTagSlot[] = []; + // Element IDs of slots TS defined itself this call. GPT requires a + // display() call to register/render a freshly-defined slot; refresh() + // alone no-ops for a slot that was never displayed, so these are + // display()ed instead of refreshed. + const slotsToDisplay: string[] = []; const divToSlotId: Record = {}; const prevSlotTargetingKeys = ts.prevSlotTargetingKeys ?? {}; const nextSlotTargetingKeys: Record = {}; @@ -468,8 +474,12 @@ export function installTsAdInit(): void { const slotTargetingKeys = Object.keys(slot.targeting ?? {}); nextSlotTargetingKeys[actualDivId] = slotTargetingKeys; if (slotDivId2 !== actualDivId) nextSlotTargetingKeys[slotDivId2] = slotTargetingKeys; - if (tsOwned) newSlots.push(gptSlot); - slotsToRefresh.push(gptSlot); + if (tsOwned) { + newSlots.push(gptSlot); + slotsToDisplay.push(slotDivId2); + } else { + slotsToRefresh.push(gptSlot); + } // APS: signal to apstag that bids are ready so Amazon's GAM creative // can render. apstag must already be initialised on the page (which it @@ -507,12 +517,20 @@ export function installTsAdInit(): void { }); } + // Register and render TS-defined slots. GPT requires display() for a + // freshly-defined slot — without it the slot no-ops ("defineSlot was + // called without a matching display call") and misses its impression. + // Must run after enableServices(); on SPA navigation services are already + // enabled, so this runs unconditionally for any newly-defined slots. + slotsToDisplay.forEach((divId) => g.display?.(divId)); + if (slotsToRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied - // server-side targeting to GAM. If slim-Prebid has wrapped refresh(), - // it must pass this call straight through — not clear the targeting - // and run a duplicate client-side auction. Later publisher-initiated - // refreshes of the same slots still go through the wrapper normally. + // server-side targeting to GAM for reused publisher-owned slots. If + // slim-Prebid has wrapped refresh(), it must pass this call straight + // through — not clear the targeting and run a duplicate client-side + // auction. Later publisher-initiated refreshes of the same slots still + // go through the wrapper normally. ts.adInitRefreshInProgress = true; try { g.pubads!().refresh(slotsToRefresh); diff --git a/crates/js/lib/test/integrations/gpt/ad_init.test.ts b/crates/js/lib/test/integrations/gpt/ad_init.test.ts index f7bb53e9d..43551644a 100644 --- a/crates/js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/js/lib/test/integrations/gpt/ad_init.test.ts @@ -138,6 +138,55 @@ describe('installTsAdInit', () => { fetchSpy.mockRestore(); }); + it('displays TS-defined slots and does not include them in refresh', async () => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + // Publisher has not defined this slot, so TS defines (owns) it. + getSlots: vi.fn().mockReturnValue([]), + addEventListener: vi.fn(), + refresh: vi.fn(), + }; + const defineSlotMock = vi.fn().mockReturnValue(mockSlot); + const displayMock = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: defineSlotMock, + display: displayMock, + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(defineSlotMock).toHaveBeenCalled(); + // GPT requires display() to register/render a freshly-defined slot. + expect(displayMock).toHaveBeenCalledWith('div-atf-sidebar'); + // TS-owned slots are displayed, not refreshed (refresh() no-ops for a slot + // that was never displayed). + expect(mockPubads.refresh).not.toHaveBeenCalled(); + }); + it('sets adInitRefreshInProgress only for the duration of the internal refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), @@ -148,7 +197,9 @@ describe('installTsAdInit', () => { let flagDuringRefresh: boolean | undefined; const mockPubads = { enableSingleRequest: vi.fn(), - getSlots: vi.fn().mockReturnValue([]), + // Publisher-owned slot reused by TS, so it goes through refresh() (which + // carries the bypass flag) rather than display(). + getSlots: vi.fn().mockReturnValue([mockSlot]), addEventListener: vi.fn(), refresh: vi.fn(() => { flagDuringRefresh = (window as TestWindow).tsjs!.adInitRefreshInProgress; diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 90eb2181b..46cfe0fd3 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -28,8 +28,13 @@ // Slots TS defined itself — tracked for SPA destroy. Publisher-owned // slots are reused but never destroyed by TS on navigation. var newSlots = []; - // All slots to refresh (TS-defined + publisher-owned reused). + // Publisher-owned slots TS reused — refreshed to pick up server-side + // targeting. The publisher already display()ed these. var slotsToRefresh = []; + // Element IDs of slots TS defined itself. GPT requires display() to + // register/render a freshly-defined slot; refresh() alone no-ops for a + // slot that was never displayed, so these are display()ed instead. + var slotsToDisplay = []; slots.forEach(function (slot) { // Resolve actual div ID: exact match first, then safe prefix scan. // div_id in config may be a stable prefix (e.g. "ad-header-0-") when @@ -93,8 +98,13 @@ if (slotElementId && slotElementId !== actualDivId) { divToSlotId[slotElementId] = slot.id; } - if (tsOwned) newSlots.push(s); - slotsToRefresh.push(s); + if (tsOwned) { + newSlots.push(s); + var displayId = s.getSlotElementId() || actualDivId; + slotsToDisplay.push(displayId); + } else { + slotsToRefresh.push(s); + } }); ts.prevGptSlots = newSlots; ts.divToSlotId = divToSlotId; @@ -103,6 +113,13 @@ googletag.enableServices(); ts.servicesEnabled = true; } + // Register and render TS-defined slots. GPT requires display() for a + // freshly-defined slot; without it the slot no-ops and misses its + // impression. Runs after enableServices(); on SPA navigation services are + // already enabled, so this runs unconditionally for new slots. + slotsToDisplay.forEach(function (divId) { + googletag.display(divId); + }); if (slotsToRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied // server-side targeting to GAM. If slim-Prebid has already wrapped From fa1410011563ac9648c2a21b6ec1cad8a0a76b37 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 09:26:48 +0530 Subject: [PATCH 103/395] Validate creative-opportunity glob patterns at build time The build-time validator only checked for a non-empty page_patterns string, so a config like page_patterns = ["["] passed the release build and then failed settings load at runtime when compile_patterns rejected the slot. Compile each pattern with the same glob::Pattern::new + ** -> * normalization contract used by the runtime compile_patterns, requiring at least one pattern that compiles. Adds glob as a build-dependency and tests for an uncompilable pattern and the recursive ** case. --- crates/trusted-server-core/Cargo.toml | 1 + .../src/creative_slot_build_check.rs | 48 +++++++++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index 6e2cbd82f..b86b48dbd 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -55,6 +55,7 @@ edgezero-core = { workspace = true } config = { workspace = true } derive_more = { workspace = true } error-stack = { workspace = true } +glob = { workspace = true } http = { workspace = true } log = { workspace = true } regex = { workspace = true } diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index 55d17f918..9970e0d28 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -22,6 +22,17 @@ fn is_valid_slot_id(id: &str) -> bool { .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') } +/// Returns `true` when `pattern` compiles as a glob, mirroring the runtime +/// `CreativeOpportunitySlot::compile_patterns` contract: try `glob::Pattern::new` +/// directly, then fall back to the `**` -> `*` normalization. A pattern that +/// fails both is dropped at runtime, leaving the slot unmatchable, so the build +/// must reject it too. +fn pattern_compiles(pattern: &str) -> bool { + glob::Pattern::new(pattern) + .or_else(|_| glob::Pattern::new(&pattern.replace("**", "*"))) + .is_ok() +} + /// Validate a single raw creative-opportunity slot. /// /// Mirrors the runtime checks in `CreativeOpportunitySlot::validate_runtime`: @@ -51,18 +62,22 @@ pub(crate) fn validate_creative_slot( )); } - // At least one non-empty page pattern. + // At least one page pattern that is non-empty and compiles as a glob. + // Runtime preparation drops uncompilable patterns and rejects the slot when + // none remain, so a private/env config like `page_patterns = ["["]` would + // otherwise pass the build and fail settings load on the deployed service. let has_valid_pattern = slot .get("page_patterns") .and_then(serde_json::Value::as_array) .is_some_and(|patterns| { patterns .iter() - .any(|p| p.as_str().is_some_and(|s| !s.trim().is_empty())) + .filter_map(serde_json::Value::as_str) + .any(|s| !s.trim().is_empty() && pattern_compiles(s)) }); if !has_valid_pattern { return Err(format!( - "slot `{id}` must include at least one non-empty page pattern" + "slot `{id}` must include at least one valid page pattern" )); } @@ -174,6 +189,33 @@ mod tests { assert!(validate_creative_slot(&slot, "123456789").is_err()); } + #[test] + fn rejects_uncompilable_glob_pattern() { + // `[` is an unterminated character class; it fails to compile both + // directly and after the ** -> * normalization, so the slot would be + // unmatchable at runtime. + let slot = json!({ + "id": "atf", + "page_patterns": ["["], + "formats": [{ "width": 300, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("uncompilable glob pattern must fail at build time"); + assert!(err.contains("valid page pattern"), "got: {err}"); + } + + #[test] + fn accepts_recursive_glob_pattern() { + // `/20**` fails direct glob compilation but compiles after the + // ** -> * normalization, matching runtime behavior. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }] + }); + assert!(validate_creative_slot(&slot, "123456789").is_ok()); + } + #[test] fn rejects_blank_gam_unit_path_override() { let slot = json!({ From 8f13d5f808676aedb204787a0383fe3a3a1ef869 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 09:26:48 +0530 Subject: [PATCH 104/395] Match Cache-Control privacy directives case-insensitively finalize_response checked for lowercase "private"/"no-store" substrings, but Cache-Control directives are case-insensitive (RFC 9111). A Cache-Control: No-Store on a Set-Cookie response was treated as cacheable and downgraded to the weaker private, max-age=0, and a Cache-Control: Private did not block operator response_headers from re-enabling shared caching. Lowercase the header value before matching. Add mixed-case No-Store / Private tests. --- .../trusted-server-adapter-fastly/src/main.rs | 4 ++ .../src/route_tests.rs | 52 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index c71ea9cde..7d81c3aa3 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -807,10 +807,13 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: // net covers ordinary navigations whose sole per-user payload is the cookie. // Skip when the response is already uncacheable so we don't clobber a // stricter directive (e.g. `no-store`). + // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match + // against a lowercased copy — `No-Store` / `Private` must count. let already_uncacheable = response .headers() .get(header::CACHE_CONTROL) .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) .is_some_and(|v| v.contains("private") || v.contains("no-store")); if !already_uncacheable && response.headers().contains_key(header::SET_COOKIE) { response.headers_mut().insert( @@ -829,6 +832,7 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: .headers() .get(header::CACHE_CONTROL) .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) .is_some_and(|v| v.contains("private")); for (key, value) in &settings.response_headers { diff --git a/crates/trusted-server-adapter-fastly/src/route_tests.rs b/crates/trusted-server-adapter-fastly/src/route_tests.rs index c616223bc..9b987a843 100644 --- a/crates/trusted-server-adapter-fastly/src/route_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/route_tests.rs @@ -1152,6 +1152,58 @@ fn finalize_response_leaves_stricter_no_store_untouched() { ); } +#[test] +fn finalize_response_treats_mixed_case_no_store_as_uncacheable() { + // Cache-Control directives are case-insensitive: `No-Store` on a Set-Cookie + // response must be recognized as already-uncacheable and left untouched, not + // downgraded to the weaker `private, max-age=0`. + let settings = create_test_settings(); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "No-Store") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("No-Store"), + "mixed-case No-Store must be treated as uncacheable and preserved" + ); +} + +#[test] +fn finalize_response_mixed_case_private_blocks_operator_surrogate_reenable() { + // A mixed-case `Private` directive must still mark the response private so + // operator response_headers cannot re-enable shared caching. + let mut settings = create_test_settings(); + settings + .response_headers + .insert("Surrogate-Control".to_string(), "max-age=86400".to_string()); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "Private, max-age=0") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|v| v.to_str().ok()), + None, + "operator Surrogate-Control must not re-enable caching for a mixed-case Private response" + ); +} + #[test] fn finalize_response_cookie_net_blocks_operator_surrogate_reenable() { // Operator response_headers must not re-add surrogate caching once the From f997c66ee6c5f94c7833fc4618ab4e78a682afca Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 09:26:48 +0530 Subject: [PATCH 105/395] Align checked-in creative auction timeout with its 500ms guidance The comment recommends a 500ms default because the value bounds the DOMContentLoaded/window.load slip, but the checked-in value was 1500ms, so a first rollout that enables slots while inheriting the default would impose a 1.5s close-body hold on cache-hit pages. Set the sample default to 500ms. --- trusted-server.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trusted-server.toml b/trusted-server.toml index 0bd461b43..dc64d468a 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -357,7 +357,7 @@ gam_network_id = "123456789" # drains in <50 ms but the auction runs to the limit. 500 ms is the recommended # default; raise only if your SSPs need more headroom and your analytics confirm # the DCL slip is acceptable. -auction_timeout_ms = 1500 # override via TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__AUCTION_TIMEOUT_MS +auction_timeout_ms = 500 # override via TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__AUCTION_TIMEOUT_MS price_granularity = "dense" # No slot templates are enabled in the checked-in default config. Add From 3a5c4b4b06668c737f36634bce1d572048c3578e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 106/395] Correct float-truncation under-bucketing in price_bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many two-decimal CPMs are not exactly representable in binary floating point: 0.29 * 100.0 is 28.999…, so flooring truncated it to 28 ("0.28"), and 1.15 became "1.14". These values feed hb_pb targeting keys, so the auction reported a cent low. Convert CPM to whole cents through a helper that nudges values sitting an ULP below a cent boundary up before flooring, leaving genuinely sub-cent values (0.015 -> "0.01") untouched. Adds a float-boundary regression test. --- .../trusted-server-core/src/price_bucket.rs | 50 +++++++++++++------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/crates/trusted-server-core/src/price_bucket.rs b/crates/trusted-server-core/src/price_bucket.rs index 8fc4e50e6..30b7430de 100644 --- a/crates/trusted-server-core/src/price_bucket.rs +++ b/crates/trusted-server-core/src/price_bucket.rs @@ -11,30 +11,40 @@ pub enum PriceGranularity { Auto, } +/// Convert a CPM in dollars to whole cents, flooring to the cent. +/// +/// Multiplying by 100 and flooring directly under-buckets common CPMs because +/// many two-decimal values are not exactly representable in binary floating +/// point: `0.29 * 100.0` is `28.999…`, which would truncate to `28` ("0.28"). +/// A tiny epsilon corrects values sitting an ULP below a cent boundary without +/// promoting genuinely sub-cent values — `0.015` (`1.4999…`) still floors to +/// `1` ("0.01"), while `0.29` correctly yields `29`. +fn cpm_to_cents(cpm: f64) -> u64 { + const CENT_EPSILON: f64 = 1e-6; + (cpm * 100.0 + CENT_EPSILON).floor() as u64 +} + #[must_use] pub fn price_bucket(cpm: f64, granularity: PriceGranularity) -> String { - // Reject NaN / Inf early so the `(x * 100.0).floor() as u64` cast below - // can never see a non-finite value (the cast's behaviour for NaN/Inf is - // implementation-defined in Rust and "saturate to 0" only by convention). + // Reject NaN / Inf early so the cast in `cpm_to_cents` can never see a + // non-finite value (the cast's behaviour for NaN/Inf is implementation- + // defined in Rust and "saturate to 0" only by convention). if !cpm.is_finite() || cpm <= 0.0 { return "0.00".to_string(); } match granularity { PriceGranularity::Low => { - let capped = cpm.min(5.0); - let cents = (capped * 100.0).floor() as u64; + let cents = cpm_to_cents(cpm.min(5.0)); let bucketed_cents = (cents / 50) * 50; format!("{:.2}", bucketed_cents as f64 / 100.0) } PriceGranularity::Medium => { - let capped = cpm.min(20.0); - let cents = (capped * 100.0).floor() as u64; + let cents = cpm_to_cents(cpm.min(20.0)); let bucketed_cents = (cents / 10) * 10; format!("{:.2}", bucketed_cents as f64 / 100.0) } PriceGranularity::High => { - let capped = cpm.min(20.0); - let cents = (capped * 100.0).floor() as u64; + let cents = cpm_to_cents(cpm.min(20.0)); format!("{:.2}", cents as f64 / 100.0) } PriceGranularity::Dense | PriceGranularity::Auto => dense_bucket(cpm), @@ -46,17 +56,14 @@ fn dense_bucket(cpm: f64) -> String { return "20.00".to_string(); } if cpm >= 8.0 { - let cents = (cpm * 100.0).floor() as u64; - let bucketed_cents = (cents / 50) * 50; + let bucketed_cents = (cpm_to_cents(cpm) / 50) * 50; return format!("{:.2}", bucketed_cents as f64 / 100.0); } if cpm >= 3.0 { - let cents = (cpm * 100.0).floor() as u64; - let bucketed_cents = (cents / 5) * 5; + let bucketed_cents = (cpm_to_cents(cpm) / 5) * 5; return format!("{:.2}", bucketed_cents as f64 / 100.0); } - let cents = (cpm * 100.0).floor() as u64; - format!("{:.2}", cents as f64 / 100.0) + format!("{:.2}", cpm_to_cents(cpm) as f64 / 100.0) } #[cfg(test)] @@ -122,6 +129,19 @@ mod tests { ); } + #[test] + fn float_boundary_cpms_are_not_under_bucketed() { + // These two-decimal CPMs are not exactly representable in binary float + // (`0.29 * 100.0 == 28.999…`); a naive floor truncates them a cent low. + assert_eq!(price_bucket(0.29, PriceGranularity::Dense), "0.29"); + assert_eq!(price_bucket(1.15, PriceGranularity::Dense), "1.15"); + assert_eq!(price_bucket(0.29, PriceGranularity::High), "0.29"); + assert_eq!(price_bucket(1.15, PriceGranularity::High), "1.15"); + // Genuinely sub-cent values must still floor, not round up. + assert_eq!(price_bucket(0.289, PriceGranularity::High), "0.28"); + assert_eq!(price_bucket(0.015, PriceGranularity::Dense), "0.01"); + } + #[test] fn non_finite_cpm_returns_zero_bucket() { for granularity in [ From 0f241cad79370d5c0ac4435f3680c39e41a4e924 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 107/395] Cap synchronous mediator timeout to its configured budget run_parallel_mediation gave the mediator the full remaining auction budget, while the dispatched collect path bounds it by remaining.min(mediator.timeout_ms()). Apply the same cap for symmetry between the two paths. --- crates/trusted-server-core/src/auction/orchestrator.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index dc3bb5e83..48aebaa97 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -243,7 +243,9 @@ impl AuctionOrchestrator { let mediator_context = AuctionContext { settings: context.settings, request: context.request, - timeout_ms: remaining_ms, + // Bound by both the remaining auction budget and the mediator's + // own configured timeout, matching the dispatched collect path. + timeout_ms: remaining_ms.min(mediator.timeout_ms()), provider_responses: Some(&provider_responses), services: context.services, }; From 64ecc74b08edce9173b8733701236a3883736a71 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 108/395] Warn when a dispatched auction is dropped on non-streaming routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit should_run_auction is decided from request signals before the origin content-type/status/encoding is known. A navigation that dispatched SSP bid requests but then routes to PassThrough (2xx non-HTML) or BufferedUnmodified (non-2xx, unsupported encoding, empty host) dropped the DispatchedAuction without collecting it — wasted SSP quota with no visibility. Log a warning on those arms when an auction was dispatched. --- crates/trusted-server-core/src/publisher.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 3c89b923d..2bcb7a751 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1346,6 +1346,17 @@ pub async fn handle_publisher_request( content_type, status, ); + if dispatched_auction.is_some() { + // should_run_auction is decided from request signals before the + // origin content-type is known. A pass-through (2xx non-HTML) + // response has no `` to inject bids into, so the dispatched + // SSP requests are wasted — surface it for quota observability. + log::warn!( + "Server-side auction dispatched but response routed to pass-through (Content-Type: '{}', status: {}); in-flight SSP bid requests will not be collected", + content_type, + status, + ); + } let (parts, body) = response.into_parts(); let response = Response::from_parts(parts, EdgeBody::empty()); Ok(PublisherResponse::PassThrough { response, body }) @@ -1368,6 +1379,16 @@ pub async fn handle_publisher_request( status, ); } + if dispatched_auction.is_some() { + // Same wasted-dispatch case as the pass-through arm: an + // unprocessable/non-2xx response can't carry injected bids, so + // the in-flight SSP requests are left uncollected. + log::warn!( + "Server-side auction dispatched but response routed to buffered-unmodified (Content-Type: '{}', status: {}); in-flight SSP bid requests will not be collected", + content_type, + status, + ); + } Ok(PublisherResponse::Buffered(response)) } ResponseRoute::Stream => { From ac0add3b9f7278c315981e902dd7a1c470386fd1 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 109/395] Test env-injected creative-opportunity slot-id rejection Lock in that a TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT override with an invalid id is rejected through from_toml_and_env, complementing the existing TOML-path test and exercising the same validation the build-time check uses. --- crates/trusted-server-core/src/settings.rs | 46 ++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 04172a9dc..bfbdc329a 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -4697,6 +4697,52 @@ formats = [{ width = 300, height = 250 }] ); } + #[test] + fn settings_rejects_env_injected_invalid_creative_opportunity_slot_id() { + // A TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT override must go through + // the same runtime slot validation as a TOML-defined slot, so an invalid + // id injected via env is rejected by from_toml_and_env (the build-time + // path uses the same validation against the merged config). + let toml = r#" +[[handlers]] +path = "^/_ts/admin" +username = "admin" +password = "unit-test-admin-secret" + +[publisher] +domain = "example.com" +cookie_domain = ".example.com" +origin_url = "https://origin.example.com" +proxy_secret = "secret" + +[ec] +passphrase = "test-secret-key-32-bytes-minimum" + +[creative_opportunities] +gam_network_id = "21765378893" +"#; + let slot_key = format!( + "{}{}CREATIVE_OPPORTUNITIES{}SLOT", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + temp_env::with_var( + slot_key, + Some( + r#"[{"id":"bad id","page_patterns":["/"],"formats":[{"width":300,"height":250}]}]"#, + ), + || { + let err = Settings::from_toml_and_env(toml) + .expect_err("should reject env-injected invalid slot id"); + assert!( + format!("{err:?}").contains("Invalid creative opportunity slot config"), + "error should mention the invalid slot id, got: {err:?}" + ); + }, + ); + } + fn creative_opportunity_settings_toml(slot_body: &str) -> String { format!( r#" From c324e09c5fc937a5986255cbaa4ac4bcc66c7094 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 110/395] Use a valid glob as the page-pattern doc example "/20**" is an invalid glob that only matches via the **->* normalization fallback; using it as the canonical example invites copy-paste of broken config. Show "/2024/*" as the primary example and keep the normalization note as the edge-case caveat. --- .../trusted-server-core/src/creative_opportunities.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 67728bd28..cf2b401d6 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -158,11 +158,12 @@ impl CreativeOpportunitySlot { /// Returns `true` if `path` matches any of this slot's [`page_patterns`](Self::page_patterns). /// - /// Patterns use glob syntax (e.g., `"/20**"` matches any path starting with `/20`, - /// `"/"` matches only the root). When a pattern contains `**` in a position that the - /// glob crate considers invalid (e.g., `b**`), the `**` is normalised to `*` before - /// matching. A single `*` matches any sequence of characters including path separators - /// because `require_literal_separator` is `false`. + /// Patterns use glob syntax (e.g., `"/2024/*"` matches any path under `/2024/`, + /// `"/"` matches only the root). A single `*` matches any sequence of characters + /// including path separators because `require_literal_separator` is `false`. + /// When a pattern contains `**` in a position the glob crate considers invalid + /// (e.g., `"/20**"` or `"b**"`), the `**` is normalised to `*` before matching — + /// prefer a valid single-`*` pattern over relying on this fallback. /// /// Patterns that cannot be compiled even after normalisation are silently skipped. #[must_use] From bdb00f56c27bf0d9d40c13ec2bf6eabfc21f0db6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 111/395] Remove dead test-only parse_ts_eids_cookie helper parse_ts_eids_cookie was gated to #[cfg(test)] and exercised only by its own tests; production reads the ts-eids cookie through resolve_client_auction_eids -> parse_prebid_eids_cookie (which enforces its own size/length caps). Remove the function, its tests, and the now-orphaned cfg(test) imports/helpers. --- crates/trusted-server-core/src/cookies.rs | 109 ---------------------- 1 file changed, 109 deletions(-) diff --git a/crates/trusted-server-core/src/cookies.rs b/crates/trusted-server-core/src/cookies.rs index 9ad09926d..a002d9c8f 100644 --- a/crates/trusted-server-core/src/cookies.rs +++ b/crates/trusted-server-core/src/cookies.rs @@ -9,12 +9,8 @@ use error_stack::{Report, ResultExt}; use http::header; use http::Request; -#[cfg(test)] -use crate::constants::COOKIE_TS_EIDS; use crate::constants::{COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_US_PRIVACY}; use crate::error::TrustedServerError; -#[cfg(test)] -use base64::{engine::general_purpose::STANDARD, Engine as _}; /// Cookie names carrying privacy consent signals. /// @@ -73,42 +69,6 @@ pub fn handle_request_cookies( } } -/// Parse Extended User IDs from the [`COOKIE_TS_EIDS`] cookie. -/// -/// The cookie value is a standard-base64-encoded JSON array of -/// [`crate::openrtb::Eid`] objects written by the Trusted Server JS SDK via -/// `btoa(JSON.stringify(eids))`. -/// -/// Returns `None` if the cookie is absent, base64-malformed, JSON-malformed, -/// or the decoded array is empty. Parse failures are logged at `debug` level -/// so operators can diagnose JS SDK / server mismatches. -#[cfg(test)] -#[must_use] -pub(crate) fn parse_ts_eids_cookie(jar: Option<&CookieJar>) -> Option> { - let value = jar?.get(COOKIE_TS_EIDS)?.value().to_owned(); - let decoded = match STANDARD.decode(&value) { - Ok(b) => b, - Err(e) => { - log::debug!("ts-eids cookie: base64 decode failed: {e}"); - return None; - } - }; - match serde_json::from_slice::>(&decoded) { - Ok(eids) if !eids.is_empty() => { - if eids.len() > 32 || eids.iter().any(|e| e.uids.len() > 32) { - log::debug!("ts-eids cookie: too many eids or uids, rejecting"); - return None; - } - Some(eids) - } - Ok(_) => None, - Err(e) => { - log::debug!("ts-eids cookie: JSON parse failed: {e}"); - None - } - } -} - /// Strips named cookies from a `Cookie` header value string. /// /// Parses the semicolon-separated cookie pairs, filters out any whose name @@ -448,73 +408,4 @@ mod tests { let stripped = strip_cookies(header, CONSENT_COOKIE_NAMES); assert_eq!(stripped, "session=abc=123=def"); } - - fn make_jar_with(name: &str, value: &str) -> CookieJar { - parse_cookies_to_jar(&format!("{name}={value}")) - } - - fn encode_eids(eids: &[serde_json::Value]) -> String { - use base64::{engine::general_purpose::STANDARD, Engine as _}; - STANDARD.encode(serde_json::to_string(eids).expect("should serialize eids")) - } - - #[test] - fn parse_ts_eids_cookie_returns_eids_for_valid_input() { - let encoded = encode_eids(&[serde_json::json!({ - "source": "id5-sync.com", - "uids": [{"id": "abc123", "atype": 1}] - })]); - let jar = make_jar_with(COOKIE_TS_EIDS, &encoded); - let eids = parse_ts_eids_cookie(Some(&jar)).expect("should parse valid ts-eids cookie"); - assert_eq!(eids.len(), 1, "should return one EID"); - assert_eq!(eids[0].source, "id5-sync.com", "should preserve source"); - assert_eq!(eids[0].uids[0].id, "abc123", "should preserve uid"); - } - - #[test] - fn parse_ts_eids_cookie_returns_none_when_cookie_absent() { - let jar = CookieJar::new(); - assert!( - parse_ts_eids_cookie(Some(&jar)).is_none(), - "should return None when cookie absent" - ); - } - - #[test] - fn parse_ts_eids_cookie_returns_none_for_empty_array() { - let encoded = encode_eids(&[]); - let jar = make_jar_with(COOKIE_TS_EIDS, &encoded); - assert!( - parse_ts_eids_cookie(Some(&jar)).is_none(), - "should return None for empty EID array" - ); - } - - #[test] - fn parse_ts_eids_cookie_returns_none_for_corrupt_base64() { - let jar = make_jar_with(COOKIE_TS_EIDS, "not!!valid!!base64"); - assert!( - parse_ts_eids_cookie(Some(&jar)).is_none(), - "should return None for corrupt base64" - ); - } - - #[test] - fn parse_ts_eids_cookie_returns_none_for_invalid_json() { - use base64::{engine::general_purpose::STANDARD, Engine as _}; - let encoded = STANDARD.encode(b"this is not json"); - let jar = make_jar_with(COOKIE_TS_EIDS, &encoded); - assert!( - parse_ts_eids_cookie(Some(&jar)).is_none(), - "should return None for invalid JSON" - ); - } - - #[test] - fn parse_ts_eids_cookie_returns_none_for_none_jar() { - assert!( - parse_ts_eids_cookie(None).is_none(), - "should return None when jar is None" - ); - } } From 83620ab0e11d0d5d0b0f6854f05ab233e1ce8afd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 17 Jun 2026 13:25:45 +0530 Subject: [PATCH 112/395] Force EC Set-Cookie responses to stay shared-uncacheable finalize_response applies the cookie cache-privacy downgrade on the HttpResponse, but the EC identity cookie is written later by ec_finalize_response onto the converted Fastly response. A first-visit navigation whose only per-user payload is the EC cookie therefore kept any public/surrogate cache headers from the origin or operator response headers, so a shared cache could store and replay one visitor's EC cookie to others. Re-apply the downgrade with enforce_set_cookie_cache_privacy after EC finalization in both the buffered and streaming branches, mirror it in the route test helper, and cover the first-visit ordering with route tests. --- .../trusted-server-adapter-fastly/src/main.rs | 31 ++++++++ .../src/route_tests.rs | 71 +++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index e7f8a71a3..943ff94cd 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -253,6 +253,9 @@ fn main() { &mut fastly_resp, ); } + // EC finalization may have just added the identity Set-Cookie, which + // the HttpResponse-stage cache guard could not see. + enforce_set_cookie_cache_privacy(&mut fastly_resp); request_filter_effects.apply_to_fastly_response(&mut fastly_resp); fastly_resp.send_to_client(); @@ -281,6 +284,9 @@ fn main() { &mut fastly_resp, ); } + // EC finalization may have just added the identity Set-Cookie, which + // the HttpResponse-stage cache guard could not see. + enforce_set_cookie_cache_privacy(&mut fastly_resp); request_filter_effects.apply_to_fastly_response(&mut fastly_resp); let mut streaming_body = fastly_resp.stream_to_client(); let mut stream_succeeded = false; @@ -906,6 +912,31 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: } } +/// Forces cookie-bearing Fastly responses to stay private to shared caches. +/// +/// [`finalize_response`] applies this same downgrade on the [`HttpResponse`], +/// but the EC identity cookie is written later by [`ec_finalize_response`] onto +/// the converted [`FastlyResponse`], so the earlier guard never sees it. +/// Re-apply it here so a first-visit navigation whose only per-user payload is +/// the EC `Set-Cookie` can never be served with `public`/surrogate cache headers +/// inherited from the origin or operator response headers — a shared cache must +/// not be able to store and replay one visitor's EC cookie to others. +/// +/// Idempotent: a response already marked `private`/`no-store` is left untouched +/// so a stricter directive is never weakened. +fn enforce_set_cookie_cache_privacy(response: &mut FastlyResponse) { + let already_uncacheable = response + .get_header_str("cache-control") + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")); + if already_uncacheable || response.get_header("set-cookie").is_none() { + return; + } + response.set_header("cache-control", "private, max-age=0"); + response.remove_header("surrogate-control"); + response.remove_header("fastly-surrogate-control"); +} + fn http_error_response(report: &Report) -> HttpResponse { let root_error = report.current_context(); log::error!("Error occurred: {:?}", report); diff --git a/crates/trusted-server-adapter-fastly/src/route_tests.rs b/crates/trusted-server-adapter-fastly/src/route_tests.rs index 2048816fc..11a32c8c0 100644 --- a/crates/trusted-server-adapter-fastly/src/route_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/route_tests.rs @@ -638,6 +638,7 @@ fn route_result_to_fastly_response( &mut fastly_response, ); } + super::enforce_set_cookie_cache_privacy(&mut fastly_response); request_filter_effects.apply_to_fastly_response(&mut fastly_response); fastly_response } @@ -1463,6 +1464,76 @@ fn finalize_response_makes_cookie_bearing_responses_private() { ); } +#[test] +fn ec_set_cookie_added_after_finalize_downgrades_origin_public_cache() { + // First-visit navigation: the origin response is shared-cacheable and carries + // no cookie, so the HttpResponse-stage finalizer keeps its cache headers. EC + // finalization then mints the identity Set-Cookie on the converted Fastly + // response, after that guard has already run. The post-EC privacy guard must + // downgrade caching so a shared cache cannot replay one visitor's EC cookie. + let settings = create_test_settings(); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "public, max-age=3600") + .header("surrogate-control", "max-age=86400") + .body(EdgeBody::empty()) + .expect("should build test response"); + + // No cookie at this stage, so the cookie net does not fire and the origin + // cache directive survives finalize_response — reproducing the gap. + super::finalize_response(&settings, None, &mut response); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("public, max-age=3600"), + "a cookieless response should keep its origin cache directive" + ); + + let mut fastly_response = compat::to_fastly_response(response); + // Stand in for ec_finalize_response minting the first-visit identity cookie: + // its EcContext constructors are #[cfg(test)] in trusted-server-core and are + // not reachable from this crate, but the only behavior under test here is the + // post-EC ordering — a Set-Cookie appearing after finalize_response ran. + fastly_response.set_header(header::SET_COOKIE, "ec=abc; Path=/; HttpOnly"); + + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("private, max-age=0"), + "an EC Set-Cookie added after finalize_response must downgrade caching" + ); + assert!( + fastly_response.get_header("surrogate-control").is_none(), + "EC Set-Cookie responses must not retain surrogate cacheability" + ); +} + +#[test] +fn enforce_set_cookie_cache_privacy_keeps_stricter_no_store() { + // A stricter directive minted alongside the cookie must not be weakened to + // the `private, max-age=0` downgrade. + let mut fastly_response = compat::to_fastly_response( + edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"), + ); + + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("no-store"), + "an already-uncacheable response should keep its stricter directive" + ); +} + #[test] fn finalize_response_leaves_stricter_no_store_untouched() { let settings = create_test_settings(); From 0cf84e4634d9fcec53e1c4cbc65778524892b60f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 17 Jun 2026 13:29:55 +0530 Subject: [PATCH 113/395] Preserve server-side bidder params on Prebid refresh auctions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The synthetic refresh ad unit only carried the trustedServer bid with a zone, so the requestBids shim had no original server-side bidder entries to collect into bidderParams. Refresh/scroll /auction requests therefore sent {} for inline PBS params and dropped demand the publisher configured only on the initial ad unit. Recover the matching original pbjs.adUnits server-side params by ad unit code — from both raw bidder entries and params already folded onto the initial trustedServer bid — and attach them to the synthetic refresh bid. --- .../js/lib/src/integrations/prebid/index.ts | 55 +++++++- .../test/integrations/prebid/index.test.ts | 124 ++++++++++++++++++ 2 files changed, 175 insertions(+), 4 deletions(-) diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index 61b546e5b..835d28fdb 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -372,6 +372,49 @@ function clientSideBidsForRefresh( return bids; } +/** + * Recover the publisher's inline server-side (PBS) bidder params for a slot. + * + * The synthetic refresh ad unit carries only the `trustedServer` bid, so the + * `requestBids` shim has no original server-side bidder entries to collect into + * `bidderParams` — without this, refresh/scroll `/auction` requests send `{}` + * and lose demand the publisher configured only on the initial ad unit. Source + * the params from the matching `pbjs.adUnits` entry by code, covering both + * states the initial auction can leave that entry in: + * - raw server-side bidder entries (`{ bidder, params }`) not yet folded, and + * - params already folded into that unit's `trustedServer` bid `bidderParams` + * by a prior `requestBids` call. + */ +function serverSideBidderParamsForRefresh(code: string): Record> { + const adUnits = (pbjs.adUnits ?? []) as TrustedServerAdUnit[]; + const match = adUnits.find((unit) => unit.code === code); + if (!match?.bids) return {}; + + const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); + const params: Record> = {}; + + for (const bid of match.bids) { + if (!bid?.bidder) continue; + if (bid.bidder === ADAPTER_CODE) { + // Params captured and folded onto the trustedServer bid by an earlier + // requestBids call. + const folded = (bid.params?.[BIDDER_PARAMS_KEY] ?? {}) as Record< + string, + Record + >; + for (const [bidder, bidderParams] of Object.entries(folded)) { + params[bidder] = bidderParams; + } + continue; + } + if (clientSideBidders.has(bid.bidder)) continue; + // Raw server-side bidder entry not yet folded by the shim. + params[bid.bidder] = bid.params ?? {}; + } + + return params; +} + function clearRefreshTargeting(slot: RefreshGptSlot): void { if (typeof slot.clearTargeting !== 'function') return; @@ -694,13 +737,17 @@ export function installRefreshHandler(timeoutMs = 1500): void { }; const code = refreshSlotElementId(slot) ?? 'refresh-slot'; + const tsParams: Record = zone ? { [ZONE_KEY]: zone } : {}; + // Carry the publisher's inline server-side (PBS) bidder params captured + // on the initial ad unit so refresh/scroll auctions don't drop them. + const serverSideParams = serverSideBidderParamsForRefresh(code); + if (Object.keys(serverSideParams).length > 0) { + tsParams[BIDDER_PARAMS_KEY] = serverSideParams; + } return { code, mediaTypes: { banner }, - bids: [ - { bidder: ADAPTER_CODE, params: zone ? { [ZONE_KEY]: zone } : {} }, - ...clientSideBidsForRefresh(code), - ], + bids: [{ bidder: ADAPTER_CODE, params: tsParams }, ...clientSideBidsForRefresh(code)], }; }); diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index 2ca650e18..5edad541f 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -922,6 +922,130 @@ describe('prebid/installRefreshHandler', () => { mockPbjs.adUnits = []; }); + it('preserves raw server-side bidder params in refresh ad units', () => { + // Original publisher ad unit carries an inline server-side appnexus bid that + // the initial auction has not yet folded into the trustedServer bid. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [{ bidder: 'appnexus', params: { placementId: 12345 } }], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, + }, + ], + }), + ], + }) + ); + + mockPbjs.adUnits = []; + }); + + it('recovers server-side bidder params already folded onto the original trustedServer bid', () => { + // After the initial auction, the requestBids shim has folded the publisher's + // server-side params into the original ad unit's trustedServer bid. A later + // refresh must still recover them by code. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { bidderParams: { appnexus: { placementId: 12345 } } }, + }, + ], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, + }, + ], + }), + ], + }) + ); + + mockPbjs.adUnits = []; + }); + it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { const originalRefresh = vi.fn(); const clearTargeting = vi.fn(); From 32de4aa5b8907f0b25bdb3b687c97bf16cda43a6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 17 Jun 2026 13:38:53 +0530 Subject: [PATCH 114/395] Reject build-time creative-opportunity configs the runtime can't load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The build script types price_granularity as a String and slots as raw JSON values, so values the runtime schema rejects — a price_granularity outside the PriceGranularity enum (e.g. custom), or unknown slot keys under the slot's deny_unknown_fields — embedded cleanly and then failed settings load on every non-health request, turning a green build into a request-time outage. Validate price_granularity against the real PriceGranularity enum and reject unknown top-level slot fields in the shared build-check validator before the merged config is embedded, with tests for both. --- crates/trusted-server-core/build.rs | 8 +- .../src/creative_slot_build_check.rs | 116 +++++++++++++++++- 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index cee32e259..f52986c8a 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -103,7 +103,7 @@ mod settings; #[path = "src/creative_slot_build_check.rs"] mod creative_slot_build_check; -use creative_slot_build_check::validate_creative_slot; +use creative_slot_build_check::{validate_creative_slot, validate_price_granularity}; use std::fs; use std::path::Path; @@ -137,6 +137,12 @@ fn main() { // `creative_slot_build_check`) so it stays under test. Running it before the // write also means a rejected config is never persisted to the embedded file. if let Some(co) = &settings.creative_opportunities { + // price_granularity is a String stub in the build context, so validate it + // against the real PriceGranularity enum before embedding — an invalid + // value would otherwise fail runtime settings load on every request. + if let Err(err) = validate_price_granularity(&co.price_granularity) { + panic!("trusted-server.toml [creative_opportunities]: {err}"); + } for slot in &co.slot_raw { if let Err(err) = validate_creative_slot(slot, &co.gam_network_id) { panic!("trusted-server.toml [creative_opportunities.slot]: {err}"); diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index 9970e0d28..6a0f446b7 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -14,6 +14,54 @@ //! intentionally empty in the build context, keeping `build.rs` free of the //! full runtime dependency graph. +/// Top-level slot fields the runtime [`CreativeOpportunitySlot`] accepts. +/// +/// The runtime struct is `#[serde(deny_unknown_fields)]`, but the build context +/// deserializes slots as raw `serde_json::Value`, which silently keeps unknown +/// keys. Mirror the runtime field set here so an env-injected typo or stray key +/// fails the build instead of failing settings load on every request. +/// +/// `compiled_patterns` is intentionally excluded: it is `#[serde(skip)]` on the +/// runtime struct and is never a valid input field. +/// +/// [`CreativeOpportunitySlot`]: crate::creative_opportunities::CreativeOpportunitySlot +const ALLOWED_SLOT_FIELDS: &[&str] = &[ + "id", + "gam_unit_path", + "div_id", + "page_patterns", + "formats", + "floor_price", + "targeting", + "providers", +]; + +/// Validate that `value` is a `price_granularity` the runtime can deserialize. +/// +/// The build context types `price_granularity` as a `String`, so an invalid +/// value such as `custom` would embed cleanly and then fail runtime settings +/// load — the real [`PriceGranularity`] enum cannot deserialize it — on every +/// non-health request. Delegating to that enum's `Deserialize` impl keeps the +/// accepted set in lockstep with the runtime, avoiding drift. +/// +/// # Errors +/// +/// Returns an error string when `value` is not one of the runtime +/// [`PriceGranularity`] variants. +/// +/// [`PriceGranularity`]: crate::price_bucket::PriceGranularity +pub(crate) fn validate_price_granularity(value: &str) -> Result<(), String> { + serde_json::from_value::(serde_json::Value::String( + value.to_string(), + )) + .map(|_| ()) + .map_err(|_| { + format!( + "price_granularity '{value}' is invalid; expected one of: low, medium, dense, high, auto" + ) + }) +} + /// Returns `true` when `id` is non-empty and only `[A-Za-z0-9_-]`. fn is_valid_slot_id(id: &str) -> bool { !id.is_empty() @@ -62,6 +110,17 @@ pub(crate) fn validate_creative_slot( )); } + // Reject unknown top-level keys, mirroring the runtime slot's + // `#[serde(deny_unknown_fields)]`. The raw-JSON build path would otherwise + // accept env-injected typos that the runtime rejects at settings load. + if let Some(object) = slot.as_object() { + for key in object.keys() { + if !ALLOWED_SLOT_FIELDS.contains(&key.as_str()) { + return Err(format!("slot `{id}` has unknown field '{key}'")); + } + } + } + // At least one page pattern that is non-empty and compiles as a glob. // Runtime preparation drops uncompilable patterns and rejects the slot when // none remain, so a private/env config like `page_patterns = ["["]` would @@ -119,9 +178,64 @@ pub(crate) fn validate_creative_slot( #[cfg(test)] mod tests { - use super::validate_creative_slot; + use super::{validate_creative_slot, validate_price_granularity}; use serde_json::json; + #[test] + fn rejects_unknown_slot_field() { + // The runtime slot is deny_unknown_fields, so an env-injected typo like + // `floorprice` must fail the build, not pass it and break settings load. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "floorprice": 1.5 + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("unknown slot field must fail at build time"); + assert!(err.contains("unknown field 'floorprice'"), "got: {err}"); + } + + #[test] + fn accepts_all_known_slot_fields() { + let slot = json!({ + "id": "atf", + "gam_unit_path": "/123456789/publisher/atf", + "div_id": "atf-div", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "floor_price": 1.5, + "targeting": { "pos": "atf" }, + "providers": {} + }); + assert!( + validate_creative_slot(&slot, "123456789").is_ok(), + "all documented slot fields must be accepted" + ); + } + + #[test] + fn accepts_valid_price_granularities() { + for value in ["low", "medium", "dense", "high", "auto"] { + assert!( + validate_price_granularity(value).is_ok(), + "'{value}' should be a valid price_granularity" + ); + } + } + + #[test] + fn rejects_invalid_price_granularity() { + // The runtime PriceGranularity enum has no `custom` variant, so a build + // that embeds it would fail settings load on every request. + let err = validate_price_granularity("custom") + .expect_err("invalid price_granularity must fail at build time"); + assert!( + err.contains("price_granularity 'custom' is invalid"), + "got: {err}" + ); + } + #[test] fn accepts_a_well_formed_slot() { let slot = json!({ From f456b506b0df3a64cd7fbb1578e82045913e551b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 17 Jun 2026 22:45:51 +0530 Subject: [PATCH 115/395] Close EC and ad-stack gaps in page-bids and publisher flow Stop handle_publisher_request from minting its own EC ID. EC generation is the adapter's real-browser-gated responsibility; the duplicate inline call re-ran for any navigation with no real-browser signal, so a non-real-browser client could get an IP-derived EC minted in memory and forwarded to PBS/APS even though the adapter blocked EC operations. Gate /__ts/page-bids slot output on the effective ad-stack condition (auction kill switch + consent), not just winning bids. Returning slots while the stack is disabled let the SPA hook run adInit() and create or refresh GPT slots client-side, defeating the kill switch. This matches the publisher navigation path's should_run_server_side_ad_stack gate. Add deny_unknown_fields to the top-level creative-opportunities config and nested provider/format structs so misspelled keys fail at startup instead of silently disabling or mis-timing the ad stack. Add regression tests for all three and update the page-bids tests to isolate the bot/prefetch variable from the consent gate. --- .../src/creative_opportunities.rs | 46 ++++ crates/trusted-server-core/src/publisher.rs | 245 ++++++++++++++---- 2 files changed, 239 insertions(+), 52 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index cf2b401d6..2b3fa6e72 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -16,6 +16,7 @@ use crate::settings::vec_from_seq_or_map; /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct CreativeOpportunitiesConfig { /// GAM network ID used to build default unit paths. pub gam_network_id: String, @@ -288,6 +289,7 @@ impl CreativeOpportunitySlot { /// An ad format combining a media type with pixel dimensions. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct CreativeOpportunityFormat { /// Creative width in pixels. pub width: u32, @@ -320,6 +322,7 @@ impl CreativeOpportunityFormat { /// Provider-specific slot identifiers for a [`CreativeOpportunitySlot`]. #[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct SlotProviders { /// Amazon Publisher Services (APS/TAM) slot parameters. pub aps: Option, @@ -333,6 +336,7 @@ pub struct SlotProviders { /// APS-specific parameters for a slot. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct ApsSlotParams { /// The APS slot ID string used when making TAM bid requests. pub slot_id: String, @@ -345,6 +349,7 @@ pub struct ApsSlotParams { /// When `bidders` is non-empty the map is forwarded verbatim, bypassing /// automatic expansion (useful for slots that need explicit per-bidder params). #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct PrebidSlotParams { /// Per-bidder inline params map. Bidder name → params object. /// @@ -595,6 +600,47 @@ mod tests { ); } + #[test] + fn config_rejects_unknown_top_level_key() { + // A typo such as `slots` instead of `slot` must surface as a config + // error rather than silently deserializing to an empty (disabled) stack. + let typo = serde_json::json!({ "gam_network_id": "12345", "slots": [] }); + assert!( + serde_json::from_value::(typo).is_err(), + "unknown top-level key should be rejected by deny_unknown_fields" + ); + + let correct = serde_json::json!({ "gam_network_id": "12345", "slot": [] }); + assert!( + serde_json::from_value::(correct).is_ok(), + "the correct `slot` key should still deserialize" + ); + } + + #[test] + fn config_rejects_unknown_nested_keys() { + // Format typo: `med.a_type` instead of `media_type`. + let format_typo = serde_json::json!({ "width": 300, "height": 250, "meda_type": "banner" }); + assert!( + serde_json::from_value::(format_typo).is_err(), + "unknown format key should be rejected" + ); + + // Provider typo: `prebd` instead of `prebid`. + let providers_typo = serde_json::json!({ "prebd": {} }); + assert!( + serde_json::from_value::(providers_typo).is_err(), + "unknown provider key should be rejected" + ); + + // APS typo: `slotId` instead of `slot_id`. + let aps_typo = serde_json::json!({ "slotId": "x" }); + assert!( + serde_json::from_value::(aps_typo).is_err(), + "unknown APS key should be rejected" + ); + } + #[test] fn prebid_slot_params_deserializes_without_bidders_field() { let json = r#"{"bidders": {}}"#; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 2bcb7a751..d19f761d3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1072,18 +1072,14 @@ pub async fn handle_publisher_request( let is_navigation = is_navigation_request(&req); - // Generate a new EC ID only for document navigations. Subresource - // requests (fonts, images, CSS) may lack consent signals such as the - // Sec-GPC header, so we skip generation to avoid setting identity - // cookies when the user's consent preference is unknown. - if is_navigation { - if let Err(err) = ec_context.generate_if_needed(settings, kv) { - log::warn!("EC generation failed: {err:?}"); - } - } else { - log::debug!("EC generation skipped: non-document request"); - } - + // EC generation is the caller's responsibility — it must run only for real + // browsers on document navigations, and that real-browser decision lives in + // the adapter (TLS/JA4/device gate). Generating here, with only the + // navigation signal, would mint an IP-derived EC for clients the adapter + // classified as non-real browsers and forward it to SSPs/APS even though EC + // operations were blocked for them. The adapter calls + // `EcContext::generate_if_needed` (real-browser-gated) before dispatching to + // this handler; subresource requests are likewise filtered there. let ec_allowed = ec_context.ec_allowed(); log::debug!( "Proxy EC state: has_ec_id={}, ec_allowed={ec_allowed}", @@ -1815,12 +1811,16 @@ pub async fn handle_page_bids( ); } - let winning_bids = if auction_enabled - && !matched_slots.is_empty() - && consent_allows_auction - && !is_bot - && !is_prefetch - { + // The [auction].enabled kill switch and a consent denial disable the entire + // server-side ad stack. In those states the endpoint must return no slots, + // so the SPA hook does not assign `ts.adSlots` and call `adInit()` — + // otherwise the kill switch/consent gate would stop SSP calls but still let + // the client create/refresh GPT slots. Bot/prefetch requests, by contrast, + // keep their slot definitions (the placement structure is unchanged) but + // skip the live auction, matching the existing bot/prefetch behaviour. + let ad_stack_enabled = auction_enabled && consent_allows_auction; + + let winning_bids = if ad_stack_enabled && !matched_slots.is_empty() && !is_bot && !is_prefetch { let slots_ctx = MatchedSlotsContext { matched_slots: &matched_slots, request_path: &path_param, @@ -1892,30 +1892,36 @@ pub async fn handle_page_bids( settings.debug.inject_adm_for_testing, ); - let slots_json: Vec = matched_slots - .iter() - .map(|slot| { - let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); - let div_id = slot.resolved_div_id(); - let formats: Vec = slot - .formats - .iter() - .map(|f| serde_json::json!([f.width, f.height])) - .collect(); - let targeting: serde_json::Map = slot - .targeting - .iter() - .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) - .collect(); - serde_json::json!({ - "id": slot.id, - "gam_unit_path": gam_path, - "div_id": div_id, - "formats": formats, - "targeting": targeting, + // Gate slots on the ad-stack kill switch / consent: when disabled, return no + // slots so the SPA hook does not call `adInit()` / create GPT slots. + let slots_json: Vec = if ad_stack_enabled { + matched_slots + .iter() + .map(|slot| { + let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); + let div_id = slot.resolved_div_id(); + let formats: Vec = slot + .formats + .iter() + .map(|f| serde_json::json!([f.width, f.height])) + .collect(); + let targeting: serde_json::Map = slot + .targeting + .iter() + .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) + .collect(); + serde_json::json!({ + "id": slot.id, + "gam_unit_path": gam_path, + "div_id": div_id, + "formats": formats, + "targeting": targeting, + }) }) - }) - .collect(); + .collect() + } else { + Vec::new() + }; let body = serde_json::json!({ "slots": slots_json, @@ -2254,6 +2260,67 @@ mod tests { ); } + #[tokio::test] + async fn handle_publisher_request_does_not_self_generate_ec() { + // EC generation is the adapter's real-browser-gated responsibility. This + // handler must never mint an EC ID on its own: for a navigation from a + // client the adapter did not pre-generate for (e.g. a non-real browser), + // `ec_value` must stay `None` so no IP-derived identifier reaches the + // auction. Consent allows EC creation and a client IP is present here — + // exactly the conditions under which the old inline call would have + // generated one. + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = + EcContext::new_for_test_with_ip(None, consent, Some("203.0.113.7".to_string())); + assert!( + ec_context.ec_allowed(), + "test precondition: consent must allow EC creation" + ); + + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/article") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + + let _ = handle_publisher_request( + &settings, + ®istry, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + ) + .await + .expect("should proxy publisher request"); + + assert_eq!( + ec_context.ec_value(), + None, + "handler must not self-generate an EC ID; generation is the adapter's real-browser-gated responsibility", + ); + } + #[test] fn test_content_type_detection() { let test_cases = vec![ @@ -3923,6 +3990,34 @@ mod tests { serde_json::from_slice(&response.into_body().into_bytes()).expect("should be json") } + /// `run_page_bids` with an EC context whose jurisdiction allows the + /// server-side auction, so slot-counting tests isolate the variable + /// under test (bot/prefetch) from the consent gate. The default + /// request resolves to `Jurisdiction::Unknown`, which fails the + /// consent gate and now suppresses slots. + async fn run_page_bids_consent_allowed( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> serde_json::Value { + let ec_context = consent_allowing_ec_context(); + let response = + run_page_bids_response_with_ec(settings, orchestrator, slots, &ec_context, req) + .await; + serde_json::from_slice(&response.into_body().into_bytes()).expect("should be json") + } + + /// Builds an [`EcContext`] whose consent context permits the server-side + /// auction (known non-GDPR jurisdiction, no EU TCF signal). + fn consent_allowing_ec_context() -> EcContext { + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + EcContext::new_for_test(None, consent) + } + fn article_slot() -> Vec { vec![CreativeOpportunitySlot { id: "atf".to_string(), @@ -3968,10 +4063,20 @@ mod tests { slots: &[CreativeOpportunitySlot], req: Request, ) -> Response { - let services = noop_services(); let fastly_req = crate::compat::to_fastly_request_ref(&req); let ec_context = EcContext::read_from_request(settings, &fastly_req) .expect("should read EC context"); + run_page_bids_response_with_ec(settings, orchestrator, slots, &ec_context, req).await + } + + async fn run_page_bids_response_with_ec( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + ec_context: &EcContext, + req: Request, + ) -> Response { + let services = noop_services(); handle_page_bids( settings, &services, @@ -3981,7 +4086,7 @@ mod tests { slots, registry: None, }, - &ec_context, + ec_context, req, ) .await @@ -4102,7 +4207,7 @@ mod tests { "Mozilla/5.0 (compatible; Googlebot/2.1)", ); - let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] @@ -4132,7 +4237,7 @@ mod tests { let mut req = make_page_bids_request("/2024/01/my-article/"); set_test_header(&mut req, "sec-purpose", "prefetch"); - let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] @@ -4210,24 +4315,27 @@ mod tests { } #[tokio::test] - async fn disabled_auction_returns_slots_but_no_bids() { - // [auction].enabled = false is a global kill switch: slot definitions - // are still returned (HTML structure unchanged) but no server-side - // auction may be dispatched. + async fn disabled_auction_returns_no_slots_or_bids() { + // [auction].enabled = false is a global kill switch: it must disable + // the entire server-side ad stack, not just SSP calls. Returning slot + // definitions would let the SPA hook assign `ts.adSlots` and call + // `adInit()`, creating/refreshing GPT slots client-side even though + // the auction is off. Consent is allowed here so the test isolates + // the kill switch. let settings = settings_with_co_auction_disabled(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let slots = article_slot(); let req = make_page_bids_request("/2024/01/my-article/"); - let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] .as_array() .expect("slots should be array") .len(), - 1, - "disabled auction should still return slot definitions" + 0, + "disabled auction must not return slot definitions (kill switch stops the ad stack)" ); assert_eq!( body["bids"] @@ -4238,5 +4346,38 @@ mod tests { "disabled auction must not produce bids" ); } + + #[tokio::test] + async fn consent_denied_returns_no_slots_or_bids() { + // When consent denies the server-side auction (here: Jurisdiction + // Unknown fails closed), the endpoint must return no slots so the SPA + // hook does not create GPT slots client-side — matching the publisher + // navigation path's `should_run_server_side_ad_stack` gate. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let req = make_page_bids_request("/2024/01/my-article/"); + + // run_page_bids uses the default EC context, which resolves to + // Jurisdiction::Unknown (consent denied). + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 0, + "consent denial must suppress slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "consent denial must produce no bids" + ); + } } } From a3160d5a33a4f2a473888e766dfb41871c130cc5 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 19 Jun 2026 23:29:03 +0530 Subject: [PATCH 116/395] Close cache-privacy and refresh-recovery gaps from PR review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply request-filter response effects before the final Set-Cookie cache guard in every Fastly response path (buffered, streaming, asset streaming) so a per-user cookie added by a DataDome allow can no longer leave with public/surrogate cache headers. Strip surrogate cache headers on every Set-Cookie response — even one keeping a stricter no-store directive — and treat no-store as protected in the operator-header guard. Reject OPTIONS /__ts/page-bids at the adapter so the side-effecting endpoint never grants a CORS preflight the publisher origin might. Drain every dispatched SSP request in the collect loop instead of breaking on the auction deadline, so a slow origin can no longer discard SSP responses that already arrived. Reject empty/whitespace div_id overrides at runtime validation, which would otherwise bind a slot to the first id-bearing DOM element. Recover Prebid refresh params and client-side bids from candidate codes ([gpt element id, injected div_id]) so container-backed slots keep the publisher's configured demand on refresh/scroll auctions. --- .../js/lib/src/integrations/prebid/index.ts | 52 +++-- .../test/integrations/prebid/index.test.ts | 68 +++++++ .../trusted-server-adapter-fastly/src/main.rs | 100 +++++++--- .../src/route_tests.rs | 185 +++++++++++++++++- .../src/auction/orchestrator.rs | 18 +- .../src/creative_opportunities.rs | 41 ++++ 6 files changed, 409 insertions(+), 55 deletions(-) diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index 835d28fdb..fe175b44d 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -342,6 +342,27 @@ function firstTargetingValue(values: string[] | undefined): string | undefined { return values?.find((value) => value.length > 0); } +/** + * Find the publisher's original `pbjs.adUnits` entry for a refreshing slot. + * + * A TS-owned GPT slot may be defined on `${div_id}-container`, so the GPT + * element id used as the synthetic refresh ad unit code can differ from the + * inner `div_id` the publisher keyed their Prebid ad unit by. Try each candidate + * code in order and return the first matching ad unit, so container-backed slots + * still recover the publisher's configured params and bidders. + */ +function findRefreshAdUnit( + candidateCodes: Array +): TrustedServerAdUnit | undefined { + const adUnits = (pbjs.adUnits ?? []) as TrustedServerAdUnit[]; + for (const code of candidateCodes) { + if (!code) continue; + const match = adUnits.find((unit) => unit.code === code); + if (match) return match; + } + return undefined; +} + /** * Collect the configured client-side bidder entries for a refreshing slot. * @@ -350,17 +371,16 @@ function firstTargetingValue(values: string[] | undefined): string | undefined { * already present on the ad unit, so without re-attaching them here publishers * that split demand between server-side and native Prebid adapters would lose * all client-side demand on refresh/scroll impressions. Bids are sourced from - * the matching `pbjs.adUnits` entry (by ad unit code) so the publisher's - * configured params are preserved. + * the matching `pbjs.adUnits` entry (by candidate ad unit code) so the + * publisher's configured params are preserved. */ function clientSideBidsForRefresh( - code: string + candidateCodes: Array ): Array<{ bidder: string; params: Record }> { const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); if (clientSideBidders.size === 0) return []; - const adUnits = (pbjs.adUnits ?? []) as TrustedServerAdUnit[]; - const match = adUnits.find((unit) => unit.code === code); + const match = findRefreshAdUnit(candidateCodes); if (!match?.bids) return []; const bids: Array<{ bidder: string; params: Record }> = []; @@ -379,15 +399,16 @@ function clientSideBidsForRefresh( * `requestBids` shim has no original server-side bidder entries to collect into * `bidderParams` — without this, refresh/scroll `/auction` requests send `{}` * and lose demand the publisher configured only on the initial ad unit. Source - * the params from the matching `pbjs.adUnits` entry by code, covering both - * states the initial auction can leave that entry in: + * the params from the matching `pbjs.adUnits` entry by candidate code, covering + * both states the initial auction can leave that entry in: * - raw server-side bidder entries (`{ bidder, params }`) not yet folded, and * - params already folded into that unit's `trustedServer` bid `bidderParams` * by a prior `requestBids` call. */ -function serverSideBidderParamsForRefresh(code: string): Record> { - const adUnits = (pbjs.adUnits ?? []) as TrustedServerAdUnit[]; - const match = adUnits.find((unit) => unit.code === code); +function serverSideBidderParamsForRefresh( + candidateCodes: Array +): Record> { + const match = findRefreshAdUnit(candidateCodes); if (!match?.bids) return {}; const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); @@ -737,17 +758,24 @@ export function installRefreshHandler(timeoutMs = 1500): void { }; const code = refreshSlotElementId(slot) ?? 'refresh-slot'; + // A TS-owned slot may be defined on `${div_id}-container`, so the GPT + // element id used as the synthetic refresh code can differ from the + // inner `div_id` the publisher keyed their ad unit by. Recover from both. + const candidateCodes = [code, injectedSlot?.div_id]; const tsParams: Record = zone ? { [ZONE_KEY]: zone } : {}; // Carry the publisher's inline server-side (PBS) bidder params captured // on the initial ad unit so refresh/scroll auctions don't drop them. - const serverSideParams = serverSideBidderParamsForRefresh(code); + const serverSideParams = serverSideBidderParamsForRefresh(candidateCodes); if (Object.keys(serverSideParams).length > 0) { tsParams[BIDDER_PARAMS_KEY] = serverSideParams; } return { code, mediaTypes: { banner }, - bids: [{ bidder: ADAPTER_CODE, params: tsParams }, ...clientSideBidsForRefresh(code)], + bids: [ + { bidder: ADAPTER_CODE, params: tsParams }, + ...clientSideBidsForRefresh(candidateCodes), + ], }; }); diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index 5edad541f..fd7703546 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -981,6 +981,74 @@ describe('prebid/installRefreshHandler', () => { mockPbjs.adUnits = []; }); + it('recovers params and client-side bids for container-backed slots by injected div_id', () => { + // A TS-owned GPT slot may be defined on `${div_id}-container`, but the + // publisher's Prebid ad unit is keyed by the inner div_id. The synthetic + // refresh code stays the GPT element id (so GPT can match it), while params + // and client-side bids are recovered from the injected div_id candidate. + (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + mockPbjs.adUnits = [ + { + code: 'div-ad-x', + bids: [ + { bidder: 'appnexus', params: { placementId: 12345 } }, + { bidder: 'rubicon', params: { accountId: 1 } }, + ], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-x-container'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'x_ad', + gam_unit_path: '/123/x', + div_id: 'div-ad-x', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + // Synthetic refresh code stays the GPT element id, not the div_id. + code: 'div-ad-x-container', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, + }, + { bidder: 'rubicon', params: { accountId: 1 } }, + ], + }), + ], + }) + ); + + delete (window as any).__tsjs_prebid; + mockPbjs.adUnits = []; + }); + it('recovers server-side bidder params already folded onto the original trustedServer bid', () => { // After the initial auction, the requestBids shim has folded the publisher's // server-side params into the original ad unit's trustedServer bid. A later diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 943ff94cd..2ef07eada 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -253,10 +253,13 @@ fn main() { &mut fastly_resp, ); } - // EC finalization may have just added the identity Set-Cookie, which - // the HttpResponse-stage cache guard could not see. - enforce_set_cookie_cache_privacy(&mut fastly_resp); + // Apply request-filter response effects (e.g. a DataDome allow + // Set-Cookie) before the final cache guard so any per-user cookie + // they add is covered. EC finalization above may also have added the + // identity Set-Cookie, which the HttpResponse-stage guard could not + // see — the guard runs last so it observes both. request_filter_effects.apply_to_fastly_response(&mut fastly_resp); + enforce_set_cookie_cache_privacy(&mut fastly_resp); fastly_resp.send_to_client(); if is_real_browser { @@ -284,10 +287,13 @@ fn main() { &mut fastly_resp, ); } - // EC finalization may have just added the identity Set-Cookie, which - // the HttpResponse-stage cache guard could not see. - enforce_set_cookie_cache_privacy(&mut fastly_resp); + // Apply request-filter response effects (e.g. a DataDome allow + // Set-Cookie) before the final cache guard so any per-user cookie + // they add is covered. EC finalization above may also have added the + // identity Set-Cookie, which the HttpResponse-stage guard could not + // see — the guard runs last so it observes both. request_filter_effects.apply_to_fastly_response(&mut fastly_resp); + enforce_set_cookie_cache_privacy(&mut fastly_resp); let mut streaming_body = fastly_resp.stream_to_client(); let mut stream_succeeded = false; match futures::executor::block_on(stream_publisher_body_async( @@ -324,7 +330,11 @@ fn main() { finalize_response(&settings, geo_info.as_ref(), &mut response); asset_cache_policy.apply_after_route_finalization(&mut response); let mut fastly_resp = compat::to_fastly_response_skeleton(response); + // A request filter (e.g. DataDome allow) can append a per-user + // Set-Cookie via response effects even on an otherwise cacheable + // asset, so guard against shared caching after applying them. request_filter_effects.apply_to_fastly_response(&mut fastly_resp); + enforce_set_cookie_cache_privacy(&mut fastly_resp); let mut streaming_body = fastly_resp.stream_to_client(); if let Err(e) = futures::executor::block_on(stream_asset_body(body, &mut streaming_body)) @@ -633,6 +643,22 @@ async fn route_request( false, ), + // Reject CORS preflight for the side-effecting page-bids endpoint at the + // adapter. The GET handler's legacy fallback trusts `X-TSJS-Page-Bids` + // precisely because this endpoint never grants a preflight; letting + // OPTIONS fall through to the publisher origin (which may return + // permissive CORS) would defeat that, allowing a cross-site page to + // trigger real PBS/APS auctions from a visitor's browser. + (Method::OPTIONS, "/__ts/page-bids") => { + let mut response = HttpResponse::new(EdgeBody::from("Forbidden")); + *response.status_mut() = edgezero_core::http::StatusCode::FORBIDDEN; + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + (Ok(response), false) + } + // SPA/CSR navigation endpoint — returns slots + bids JSON for the given path (Method::GET, "/__ts/page-bids") => ( handle_page_bids( @@ -870,34 +896,41 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: // stricter directive (e.g. `no-store`). // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match // against a lowercased copy — `No-Store` / `Private` must count. - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); - if !already_uncacheable && response.headers().contains_key(header::SET_COOKIE) { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), - ); + if response.headers().contains_key(header::SET_COOKIE) { + // Surrogate cache headers must come off every cookie-bearing response, + // even one already carrying a stricter `no-store`/`private` directive — + // they are independent of Cache-Control and would otherwise let a shared + // cache store and replay one visitor's Set-Cookie. response.headers_mut().remove("surrogate-control"); response.headers_mut().remove("fastly-surrogate-control"); + let already_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")); + if !already_uncacheable { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=0"), + ); + } } // Per-user responses (assembled HTML, page-bids, cookie-bearing navigations) - // carry a private Cache-Control directive. Operator headers must not - // re-enable shared caching for them — neither by replacing Cache-Control nor - // by reintroducing the surrogate cache headers the privacy paths stripped. - let response_is_private = response + // carry an uncacheable Cache-Control directive (`private` or `no-store`). + // Operator headers must not re-enable shared caching for them — neither by + // replacing Cache-Control nor by reintroducing the surrogate cache headers + // the privacy paths stripped. + let response_is_uncacheable = response .headers() .get(header::CACHE_CONTROL) .and_then(|v| v.to_str().ok()) .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private")); + .is_some_and(|v| v.contains("private") || v.contains("no-store")); for (key, value) in &settings.response_headers { - if response_is_private + if response_is_uncacheable && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) || key.eq_ignore_ascii_case("surrogate-control") || key.eq_ignore_ascii_case("fastly-surrogate-control")) @@ -922,19 +955,26 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: /// inherited from the origin or operator response headers — a shared cache must /// not be able to store and replay one visitor's EC cookie to others. /// -/// Idempotent: a response already marked `private`/`no-store` is left untouched -/// so a stricter directive is never weakened. +/// Idempotent: a response already marked `private`/`no-store` keeps its stricter +/// `Cache-Control`, but the surrogate cache headers are stripped regardless so a +/// `no-store` cookie response can never retain shared Fastly cacheability. fn enforce_set_cookie_cache_privacy(response: &mut FastlyResponse) { + if response.get_header("set-cookie").is_none() { + return; + } + // Strip surrogate cache headers on every cookie-bearing response, even when + // keeping a stricter `no-store`/`private` directive — Surrogate-Control is + // independent of Cache-Control and would otherwise let a shared cache store + // and replay one visitor's Set-Cookie. + response.remove_header("surrogate-control"); + response.remove_header("fastly-surrogate-control"); let already_uncacheable = response .get_header_str("cache-control") .map(str::to_ascii_lowercase) .is_some_and(|v| v.contains("private") || v.contains("no-store")); - if already_uncacheable || response.get_header("set-cookie").is_none() { - return; + if !already_uncacheable { + response.set_header("cache-control", "private, max-age=0"); } - response.set_header("cache-control", "private, max-age=0"); - response.remove_header("surrogate-control"); - response.remove_header("fastly-surrogate-control"); } fn http_error_response(report: &Report) -> HttpResponse { diff --git a/crates/trusted-server-adapter-fastly/src/route_tests.rs b/crates/trusted-server-adapter-fastly/src/route_tests.rs index 11a32c8c0..ebdec221a 100644 --- a/crates/trusted-server-adapter-fastly/src/route_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/route_tests.rs @@ -638,8 +638,11 @@ fn route_result_to_fastly_response( &mut fastly_response, ); } - super::enforce_set_cookie_cache_privacy(&mut fastly_response); + // Mirror main's ordering: apply request-filter response effects (which may + // append a per-user Set-Cookie) before the final cache guard so the guard + // observes them. request_filter_effects.apply_to_fastly_response(&mut fastly_response); + super::enforce_set_cookie_cache_privacy(&mut fastly_response); fastly_response } @@ -1534,6 +1537,134 @@ fn enforce_set_cookie_cache_privacy_keeps_stricter_no_store() { ); } +#[test] +fn enforce_set_cookie_cache_privacy_strips_surrogate_on_no_store() { + // A `no-store` cookie response keeps its stricter Cache-Control but must still + // lose the surrogate cache headers — they are independent of Cache-Control and + // would otherwise let a shared cache store and replay the visitor's cookie. + let mut fastly_response = compat::to_fastly_response( + edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .header("surrogate-control", "max-age=86400") + .header("fastly-surrogate-control", "max-age=86400") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"), + ); + + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("no-store"), + "should keep the stricter no-store directive" + ); + assert!( + fastly_response.get_header("surrogate-control").is_none(), + "no-store cookie responses must not retain Surrogate-Control" + ); + assert!( + fastly_response + .get_header("fastly-surrogate-control") + .is_none(), + "no-store cookie responses must not retain Fastly-Surrogate-Control" + ); +} + +#[test] +fn request_filter_set_cookie_after_guard_still_downgrades_cache() { + // A request filter (e.g. a DataDome allow) can append a per-user Set-Cookie via + // response effects. main applies those effects before the final cache guard, so + // an origin response still marked `public` with surrogate headers must be + // downgraded once the filter cookie is present. + use trusted_server_core::integrations::{HeaderMutation, RequestFilterEffects}; + + let mut fastly_response = compat::to_fastly_response( + edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=3600") + .header("surrogate-control", "max-age=86400") + .body(EdgeBody::empty()) + .expect("should build test response"), + ); + + let effects = RequestFilterEffects { + request_headers: vec![], + response_headers: vec![HeaderMutation::append( + "set-cookie", + "datadome=allow; Path=/; HttpOnly", + )], + }; + + // Mirror main's ordering: apply effects first, then the guard. + effects.apply_to_fastly_response(&mut fastly_response); + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("private, max-age=0"), + "a filter-added Set-Cookie must downgrade a public origin response" + ); + assert!( + fastly_response.get_header("surrogate-control").is_none(), + "a filter-added Set-Cookie must strip surrogate cacheability" + ); +} + +#[test] +fn finalize_response_no_store_cookie_blocks_operator_surrogate_reenable() { + // Operator response_headers must not re-add surrogate caching to a Set-Cookie + // response carrying the stricter `no-store` directive — the operator guard must + // treat no-store as protected, not just `private`. + let mut settings = create_test_settings(); + settings + .response_headers + .insert("Surrogate-Control".to_string(), "max-age=86400".to_string()); + settings.response_headers.insert( + "Fastly-Surrogate-Control".to_string(), + "max-age=86400".to_string(), + ); + settings.response_headers.insert( + header::CACHE_CONTROL.as_str().to_string(), + "public, max-age=3600".to_string(), + ); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "no-store") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("no-store"), + "operator Cache-Control must not weaken the stricter no-store directive" + ); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|v| v.to_str().ok()), + None, + "operator Surrogate-Control must not re-enable caching for a no-store cookie response" + ); + assert_eq!( + response + .headers() + .get("fastly-surrogate-control") + .and_then(|v| v.to_str().ok()), + None, + "operator Fastly-Surrogate-Control must not re-enable caching for a no-store cookie response" + ); +} + #[test] fn finalize_response_leaves_stricter_no_store_untouched() { let settings = create_test_settings(); @@ -1757,6 +1888,58 @@ fn page_bids_cross_site_request_is_rejected_at_the_route() { ); } +#[test] +fn page_bids_options_preflight_is_rejected_at_the_route() { + // OPTIONS must not fall through to the publisher origin (which may return + // permissive CORS); the GET handler's legacy `X-TSJS-Page-Bids` fallback + // relies on this endpoint never granting a preflight. + let base = base_route_settings_toml(); + let prebid = prebid_integration_toml(); + let config = format!( + r#"{base} + +{prebid} + + [auction] + enabled = true + providers = ["prebid"] + timeout_ms = 2000 + + [creative_opportunities] + gam_network_id = "1234" + "#, + ); + let settings = + Settings::from_toml(&config).expect("should parse page-bids route test settings"); + let (orchestrator, integration_registry) = build_route_stack(&settings); + + let req = Request::new( + Method::OPTIONS, + "https://test-publisher.com/__ts/page-bids?path=/2024/article/", + ); + let services = test_runtime_services(&req); + + let resp = route_buffered_response( + &settings, + &orchestrator, + &integration_registry, + &services, + req, + "should route page-bids preflight request", + ); + + assert_eq!( + resp.get_status(), + StatusCode::FORBIDDEN, + "should reject the page-bids CORS preflight at the adapter" + ); + assert_eq!( + resp.get_header_str(header::CACHE_CONTROL), + Some("private, no-store"), + "preflight rejection must not be shared-cached" + ); +} + #[test] fn s3_asset_origin_error_stays_uncacheable_after_global_headers() { let mut settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 48aebaa97..c654d39ee 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -936,18 +936,12 @@ impl AuctionOrchestrator { } } - // Defense-in-depth deadline guard, mirroring run_providers_parallel. - // Dispatch already caps each backend's first_byte_timeout at the - // remaining auction budget, so this should not fire in practice — - // it protects against the two paths drifting apart. - if remaining_budget_ms(auction_start, timeout_ms) == 0 && !remaining.is_empty() { - log::warn!( - "Auction timeout ({}ms) reached during collection, dropping {} remaining request(s)", - timeout_ms, - remaining.len() - ); - break; - } + // Drain every dispatched request. Each backend was capped with a + // first-byte timeout at dispatch time, so by the collect phase the + // remaining handles may already be ready even if wall-clock time + // elapsed while the origin was slow — dropping them here would + // discard SSP responses that already arrived. The mediator launch + // below still observes A_deadline via `remaining_budget_ms`. } let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 2b3fa6e72..c55d98bc0 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -143,6 +143,21 @@ impl CreativeOpportunitySlot { format.validate_runtime(&self.id)?; } + // An explicit empty/whitespace `div_id` override is rejected: the + // injected JS resolves slots with `candidate.id.startsWith(slot.div_id)`, + // and every element id starts with the empty string, so an empty override + // would bind the slot to the first id-bearing element in the document. + if self + .div_id + .as_deref() + .is_some_and(|div_id| div_id.trim().is_empty()) + { + return Err(format!( + "slot `{}` div_id override must not be empty", + self.id + )); + } + if self .resolved_gam_unit_path(gam_network_id) .trim() @@ -509,6 +524,32 @@ mod tests { assert_eq!(slot.resolved_div_id(), "atf"); } + #[test] + fn validate_runtime_rejects_empty_div_id_override() { + // An empty/whitespace div_id would resolve every slot to the first + // id-bearing element via `candidate.id.startsWith(slot.div_id)`. + let mut slot = make_slot("atf", vec!["/"]); + slot.compile_patterns(); + + slot.div_id = Some(String::new()); + assert!( + slot.validate_runtime("1234").is_err(), + "empty div_id override should fail validation" + ); + + slot.div_id = Some(" ".to_string()); + assert!( + slot.validate_runtime("1234").is_err(), + "whitespace-only div_id override should fail validation" + ); + + slot.div_id = Some("div-ad-x".to_string()); + assert!( + slot.validate_runtime("1234").is_ok(), + "a concrete div_id override should pass validation" + ); + } + #[test] fn to_ad_slot_wires_aps_params_into_bidders() { let mut slot = make_slot("atf", vec!["/"]); From bf76d41125d9bb8e09f26892ad70b993ea120293 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 20 Jun 2026 19:28:22 +0530 Subject: [PATCH 117/395] Close EID-consent and GPT initial-load gaps from PR review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate /auction client EID resolution on the same identity-consent condition as the EC ID (`ec_id.is_some()`, already filtered by `ec_allowed()`). Previously client-provided EIDs from the request body or ts-eids cookie were resolved unconditionally, so a US/GPC or US-Privacy opt-out context — where EC identity use is denied but a non-personalized auction may still run — could forward persistent EIDs, since `gate_eids_by_consent` only strips on TCF/GDPR signals. This matches the publisher and /__ts/page-bids paths. Refresh TS-defined GPT slots when the publisher disabled initial load. With pubads().disableInitialLoad(), display() only registers a freshly defined slot and the ad request must come from refresh(); TS-owned first-impression slots were only display()ed, so they rendered blank. A wrapper around disableInitialLoad() records the state on window.tsjs, and adInit() refreshes its own slots when it is set (bundle and gpt_bootstrap.js). The detector only hooks an existing googletag stub so a plain import never touches window.googletag. --- crates/js/lib/src/core/types.ts | 8 + crates/js/lib/src/integrations/gpt/index.ts | 67 +++++++- .../lib/test/integrations/gpt/ad_init.test.ts | 56 +++++++ .../src/auction/endpoints.rs | 148 +++++++++++++++++- .../src/integrations/gpt.rs | 29 ++++ .../src/integrations/gpt_bootstrap.js | 35 ++++- 6 files changed, 330 insertions(+), 13 deletions(-) diff --git a/crates/js/lib/src/core/types.ts b/crates/js/lib/src/core/types.ts index 70d40e2b6..ec2882efb 100644 --- a/crates/js/lib/src/core/types.ts +++ b/crates/js/lib/src/core/types.ts @@ -113,6 +113,14 @@ export interface TsjsApi { * client-side auction that would clear the just-applied TS targeting. */ adInitRefreshInProgress?: boolean; + /** + * True once the publisher has called `googletag.pubads().disableInitialLoad()`. + * GPT exposes no getter for this state, so it is tracked by wrapping the + * setter. When set, `display()` only registers a slot and the ad request must + * come from a `refresh()`; adInit() uses this to refresh its own freshly + * defined slots so they are not left blank. + */ + gptInitialLoadDisabled?: boolean; /** Guards SPA pushState hook installation. */ spaHookInstalled?: boolean; } diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 2effbc593..aad98c96c 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -106,6 +106,7 @@ interface GoogleTagPubAdsService { addEventListener(event: string, fn: (e: SlotRenderEndedEvent) => void): void; refresh(slots?: GoogleTagSlot[]): void; getSlots?(): GoogleTagSlot[]; + disableInitialLoad?(): void; } interface GoogleTag { @@ -371,8 +372,48 @@ function queueWinBillingBeacon(url: string): boolean { * Idempotent: destroys previously created TS-managed slots before redefining them, * so it is safe to call again after SPA navigation updates `tsjs.adSlots`/`tsjs.bids`. */ +/** + * Track whether the publisher disabled GPT initial load. + * + * GPT exposes no getter for the initial-load-disabled flag, so wrap + * `pubads().disableInitialLoad()` to record it on `window.tsjs`. With initial + * load disabled, `display()` only registers a slot — the ad request must come + * from a later `refresh()`. adInit() reads this to refresh its own freshly + * defined slots so they are not left blank. + * + * Installed via the command queue so it runs before the publisher's own + * `disableInitialLoad()` call (the TS core script is injected ahead of the + * publisher's GPT setup). Idempotent per pubads service. + * + * Only hooks an existing `googletag` stub — it never creates one. A plain module + * import that does not activate the GPT integration must not touch + * `window.googletag`. When the GPT shim is active it creates the stub before + * `installTsAdInit` runs, so the detector is still queued ahead of the + * publisher's GPT setup. + */ +function installInitialLoadDetector(ts: TsjsApi): void { + const win = window as GptWindow; + const cmd = win.googletag?.cmd; + if (!cmd) return; + cmd.push(() => { + const pubads = win.googletag?.pubads?.(); + if (!pubads) return; + const service = pubads as GoogleTagPubAdsService & { __tsInitialLoadHooked?: boolean }; + if (typeof service.disableInitialLoad !== 'function' || service.__tsInitialLoadHooked) { + return; + } + const original = service.disableInitialLoad.bind(service); + service.disableInitialLoad = function () { + ts.gptInitialLoadDisabled = true; + return original(); + }; + service.__tsInitialLoadHooked = true; + }); +} + export function installTsAdInit(): void { const ts = (window.tsjs ??= {} as TsjsApi); + installInitialLoadDetector(ts); ts.adInit = function () { const slots = ts.adSlots ?? []; // Snapshot bids at adInit() call time — correct for targeting setup. @@ -524,16 +565,28 @@ export function installTsAdInit(): void { // enabled, so this runs unconditionally for any newly-defined slots. slotsToDisplay.forEach((divId) => g.display?.(divId)); - if (slotsToRefresh.length > 0) { + // Slots needing an explicit ad request via refresh(). Reused + // publisher-owned slots always need one to pick up the just-applied + // server-side targeting. TS-defined slots are normally fetched by the + // display() above — but when the publisher called + // pubads().disableInitialLoad(), display() only registers the slot and the + // ad request must come from refresh(). Without this, a TS-owned + // first-impression slot renders blank on initial-load-disabled pages. Only + // add them in that case; otherwise display() + refresh() would + // double-request the impression. + const slotsNeedingRefresh = ts.gptInitialLoadDisabled + ? slotsToRefresh.concat(newSlots) + : slotsToRefresh; + + if (slotsNeedingRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied - // server-side targeting to GAM for reused publisher-owned slots. If - // slim-Prebid has wrapped refresh(), it must pass this call straight - // through — not clear the targeting and run a duplicate client-side - // auction. Later publisher-initiated refreshes of the same slots still - // go through the wrapper normally. + // server-side targeting to GAM. If slim-Prebid has wrapped refresh(), it + // must pass this call straight through — not clear the targeting and run + // a duplicate client-side auction. Later publisher-initiated refreshes of + // the same slots still go through the wrapper normally. ts.adInitRefreshInProgress = true; try { - g.pubads!().refresh(slotsToRefresh); + g.pubads!().refresh(slotsNeedingRefresh); } finally { ts.adInitRefreshInProgress = false; } diff --git a/crates/js/lib/test/integrations/gpt/ad_init.test.ts b/crates/js/lib/test/integrations/gpt/ad_init.test.ts index 43551644a..d649778bc 100644 --- a/crates/js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/js/lib/test/integrations/gpt/ad_init.test.ts @@ -187,6 +187,62 @@ describe('installTsAdInit', () => { expect(mockPubads.refresh).not.toHaveBeenCalled(); }); + it('refreshes TS-defined slots when the publisher disabled GPT initial load', async () => { + // With pubads().disableInitialLoad(), display() only registers a freshly + // defined slot — the ad request must come from refresh(). A TS-owned slot + // must therefore be refreshed too, or it renders blank. + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + // Publisher has not defined this slot, so TS defines (owns) it. + getSlots: vi.fn().mockReturnValue([]), + addEventListener: vi.fn(), + refresh: vi.fn(), + disableInitialLoad: vi.fn(), + }; + const displayMock = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + display: displayMock, + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + + // Publisher disables initial load — goes through the wrapper the detector + // installed, recording the state on window.tsjs. + mockPubads.disableInitialLoad(); + expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).toBe(true); + + (window as TestWindow).tsjs!.adInit!(); + + // The slot is still registered via display(), and additionally refreshed so + // it actually requests an ad under disableInitialLoad(). + expect(displayMock).toHaveBeenCalledWith('div-atf-sidebar'); + expect(mockPubads.refresh).toHaveBeenCalledWith([mockSlot]); + }); + it('sets adInitRefreshInProgress only for the duration of the internal refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 5ed59aae5..42e9d3939 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -207,10 +207,23 @@ pub async fn handle_auction( // current request does not include them, fall back to the persisted // `ts-eids` cookie so later requests can still forward the browser's // full OpenRTB-style EID structure. - let client_eids = resolve_client_auction_eids( - body.eids.as_ref(), - extract_cookie_value(&http_req, COOKIE_TS_EIDS).as_deref(), - ); + // + // Gate this on the same identity-consent condition as the EC ID + // (`ec_id.is_some()`, which is already filtered by `ec_context.ec_allowed()`). + // Otherwise a US/GPC or US-Privacy opt-out context — where EC identity use is + // denied but a non-personalized auction may still run — could forward + // persistent client EIDs from the body/cookie, since `gate_eids_by_consent` + // only strips on TCF/GDPR signals. This matches the publisher and + // `/__ts/page-bids` paths, which also resolve client EIDs only when + // `ec_id.is_some()`. + let client_eids = if ec_id.is_some() { + resolve_client_auction_eids( + body.eids.as_ref(), + extract_cookie_value(&http_req, COOKIE_TS_EIDS).as_deref(), + ) + } else { + None + }; // Resolve partner EIDs from the KV identity graph when the user has // a valid EC and both KV and partner stores are available. @@ -609,6 +622,133 @@ mod tests { ); } + /// Provider that records whether the auction request it received carried + /// EIDs, then fails its launch so no real transport handle is needed. + struct EidCapturingProvider { + had_eids: Arc>>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for EidCapturingProvider { + fn provider_name(&self) -> &'static str { + "eid_capturing_provider" + } + + async fn request_bids( + &self, + request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + *self.had_eids.lock().expect("should lock captured eids") = + Some(request.user.eids.is_some()); + Err(Report::new(TrustedServerError::Auction { + message: "capture only".to_string(), + })) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("parse_response must not run when the launch fails"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _timeout_ms: u32) -> Option { + Some("capture-backend".to_string()) + } + } + + #[tokio::test] + async fn auction_strips_client_eids_when_ec_identity_denied() { + // US-state opt-out via GPC: the server-side auction consent gate still + // allows a non-personalized auction, but EC identity use is denied + // (`ec_allowed()` is false) and `gate_eids_by_consent` does not strip + // because no TCF signal is present and GDPR does not apply. Client EIDs + // supplied in the request body/cookie must NOT be forwarded — the + // outgoing auction request must have `user.eids == None`. + let settings = create_test_settings(); + let config = AuctionConfig { + enabled: true, + providers: vec!["eid_capturing_provider".to_string()], + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + let had_eids = Arc::new(std::sync::Mutex::new(None)); + orchestrator.register_provider(Arc::new(EidCapturingProvider { + had_eids: Arc::clone(&had_eids), + })); + let services = noop_services(); + + // US-state jurisdiction with an explicit GPC opt-out: auction allowed, + // EC identity denied. + let ec_context = EcContext::new_for_test( + None, + ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + ..ConsentContext::default() + }, + ); + + // Persistent EIDs supplied in both the request body and the ts-eids cookie. + let cookie_payload = json!([ + { + "source": "sharedid.org", + "uids": [{ "id": "cookie_uid", "atype": 3 }] + } + ]); + let encoded_cookie = BASE64 + .encode(serde_json::to_vec(&cookie_payload).expect("should serialize cookie payload")); + let body = json!({ + "adUnits": [ + { + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + } + ], + "eids": [ + { + "source": "id5-sync.com", + "uids": [{ "id": "body_uid", "atype": 1 }] + } + ] + }); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .header("cookie", format!("{COOKIE_TS_EIDS}={encoded_cookie}")) + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize body"), + )) + .expect("should build auction request"); + + // The capturing provider fails its launch, so the auction errors overall; + // the assertion is on the EIDs observed by the provider, not the result. + let _ = handle_auction( + &settings, + &orchestrator, + None, + None, + &ec_context, + &services, + req, + ) + .await; + + assert_eq!( + *had_eids.lock().expect("should lock captured eids"), + Some(false), + "outgoing auction request must carry no EIDs when EC identity is denied" + ); + } + #[test] fn resolve_auction_eids_returns_none_without_kv() { let registry = PartnerRegistry::empty(); diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 118527971..e24a138d4 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -1231,6 +1231,35 @@ mod tests { ); } + #[test] + fn head_inserts_bootstrap_refreshes_ts_slots_when_initial_load_disabled() { + // Mirrors the bundle: when the publisher calls disableInitialLoad(), + // display() only registers a TS-defined slot, so the bootstrap must also + // refresh those slots or they render blank. + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let combined = integration.head_inserts(&ctx).join(""); + assert!( + combined.contains("disableInitialLoad"), + "bootstrap should wrap disableInitialLoad() to detect the disabled state" + ); + assert!( + combined.contains("gptInitialLoadDisabled"), + "bootstrap should record the initial-load-disabled state on window.tsjs" + ); + assert!( + combined.contains("slotsNeedingRefresh"), + "bootstrap should refresh TS-defined slots when initial load is disabled" + ); + } + #[test] fn head_injector_integration_id() { let integration = GptIntegration::new(test_config()); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 46cfe0fd3..f1bd9d833 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -19,6 +19,29 @@ var ts = (window.tsjs = window.tsjs || {}); if (ts.adInit) return; + // Track whether the publisher disabled GPT initial load. GPT exposes no + // getter for this, so wrap pubads().disableInitialLoad() to record it. With + // initial load disabled, display() only registers a slot and the ad request + // must come from a later refresh(); adInit() reads this to refresh its own + // freshly defined slots so they are not left blank. Pushed onto the command + // queue so it runs before the publisher's own disableInitialLoad() call. + (window.googletag = window.googletag || { cmd: [] }).cmd.push(function () { + var pubads = googletag.pubads && googletag.pubads(); + if ( + !pubads || + typeof pubads.disableInitialLoad !== "function" || + pubads.__tsInitialLoadHooked + ) { + return; + } + var original = pubads.disableInitialLoad.bind(pubads); + pubads.disableInitialLoad = function () { + ts.gptInitialLoadDisabled = true; + return original(); + }; + pubads.__tsInitialLoadHooked = true; + }); + ts.adInit = function () { var slots = ts.adSlots || []; var bids = ts.bids || {}; @@ -120,7 +143,15 @@ slotsToDisplay.forEach(function (divId) { googletag.display(divId); }); - if (slotsToRefresh.length > 0) { + // Reused publisher-owned slots always need a refresh to pick up the + // server-side targeting. TS-defined slots are fetched by display() above + // unless the publisher disabled initial load, in which case display() only + // registers them and refresh() must request the ad — otherwise they render + // blank. Only add them in that case to avoid double-requesting. + var slotsNeedingRefresh = ts.gptInitialLoadDisabled + ? slotsToRefresh.concat(newSlots) + : slotsToRefresh; + if (slotsNeedingRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied // server-side targeting to GAM. If slim-Prebid has already wrapped // refresh(), it must pass this call straight through — not clear the @@ -128,7 +159,7 @@ // bundle's adInit() in crates/js/lib/src/integrations/gpt/index.ts. ts.adInitRefreshInProgress = true; try { - googletag.pubads().refresh(slotsToRefresh); + googletag.pubads().refresh(slotsNeedingRefresh); } finally { ts.adInitRefreshInProgress = false; } From d2f538b0f3efe7daeca8facb07aaad8892b27aac Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 22 Jun 2026 16:32:51 +0530 Subject: [PATCH 118/395] Close build/runtime validation parity and observability gaps from PR review Address PR #680 review findings: Blocking build/runtime parity: - Remove the dead glob stub in build.rs so creative-slot page-pattern validation runs against the real glob crate. An invalid pattern such as `["["]` now fails the build instead of being embedded and dropped at runtime settings load. - Reject an empty/whitespace div_id override at build time, mirroring CreativeOpportunitySlot::validate_runtime. - Validate nested creative-slot fields (formats, providers, aps, prebid) against the runtime structs' deny_unknown_fields so env-injected typos like `mediatype` or `slotId` fail the build, not runtime. Observability and correctness: - Mirror the parallel auction path on the dispatch/collect path: attribute provider parse failures (error_type + message) and transport failures (via failed_backend_name) in provider_details. - Warn on each page pattern dropped during compile_patterns so a mixed valid/invalid set is visible to operators. - Escape the terminator in the configured slim_prebid_url so it cannot break out of its inline script tag. - Guard SPA navigation: onNavigate no-ops when the path is unchanged, so popstate (hash-only or same-path back/forward) no longer re-requests impressions. Docs and comments: - Update the GPT scroll/refresh handoff comment to reflect installSpaAuctionHook + /__ts/page-bids ownership of SPA navigation. - Note that targeting.zone is not forwarded when explicit prebid.bidders are set. - Split the page-bids same-origin-gate and path-normalization docs onto their own functions; remove the stale # Panics section on handle_publisher_request. - Correct the stale slotRenderEnded/beacon comment in gpt_bootstrap.js. Tests added for div_id, nested-field, slim_prebid_url escaping, and SPA same-path guard behavior. --- crates/js/lib/src/integrations/gpt/index.ts | 13 +- .../test/integrations/gpt/spa_hook.test.ts | 50 ++++- crates/trusted-server-core/build.rs | 18 +- .../src/auction/orchestrator.rs | 48 ++++- .../src/creative_opportunities.rs | 24 ++- .../src/creative_slot_build_check.rs | 186 +++++++++++++++++- .../src/integrations/gpt.rs | 56 +++++- .../src/integrations/gpt_bootstrap.js | 7 +- crates/trusted-server-core/src/publisher.rs | 16 +- 9 files changed, 369 insertions(+), 49 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index aad98c96c..30071220d 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -661,8 +661,15 @@ export function installSpaAuctionHook(): void { ts.spaHookInstalled = true; let inflight: AbortController | null = null; + // Last path an auction was run for. popstate fires for hash-only and + // same-pathname back/forward (scroll restoration), and pushState/replaceState + // can be called with the current URL, so guard every entry point against + // re-requesting impressions for a path we already loaded. + let currentPath = location.pathname; async function onNavigate(path: string): Promise { + if (path === currentPath) return; + currentPath = path; inflight?.abort(); const controller = new AbortController(); inflight = controller; @@ -696,12 +703,10 @@ export function installSpaAuctionHook(): void { function patchHistoryMethod(method: 'pushState' | 'replaceState'): void { const original = history[method].bind(history); history[method] = function (state: unknown, unused: string, url?: string | URL | null): void { - const prevPath = location.pathname; original(state, unused, url); const newPath = url ? new URL(String(url), location.href).pathname : location.pathname; - if (newPath !== prevPath) { - void onNavigate(newPath); - } + // onNavigate no-ops when newPath equals the last loaded path. + void onNavigate(newPath); }; } diff --git a/crates/js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/js/lib/test/integrations/gpt/spa_hook.test.ts index 751b081f0..6be0a8484 100644 --- a/crates/js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/js/lib/test/integrations/gpt/spa_hook.test.ts @@ -21,6 +21,12 @@ async function flushAsync(): Promise { describe('installSpaAuctionHook', () => { let fetchStub: ReturnType; + // popstate listeners registered by each module import. In production the hook + // installs once (guarded by `ts.spaHookInstalled`), but tests wipe + // `window.tsjs` and re-import per test, so without explicit removal the + // listeners accumulate on the shared window and all fire on every dispatch. + let popstateHandlers: EventListenerOrEventListenerObject[] = []; + const realAddEventListener = window.addEventListener.bind(window); beforeEach(() => { vi.resetModules(); @@ -31,6 +37,11 @@ describe('installSpaAuctionHook', () => { history.replaceState = originalReplaceState; fetchStub = vi.fn(); vi.stubGlobal('fetch', fetchStub); + popstateHandlers = []; + vi.spyOn(window, 'addEventListener').mockImplementation((type, listener, options) => { + if (type === 'popstate' && listener) popstateHandlers.push(listener); + return realAddEventListener(type, listener, options); + }); }); afterEach(() => { @@ -40,6 +51,10 @@ describe('installSpaAuctionHook', () => { originalReplaceState({}, '', '/'); // Drop any ad containers inserted by a test so DOM state does not leak. document.body.innerHTML = ''; + // Remove this test's popstate listener(s) so they do not fire in later tests. + popstateHandlers.forEach((handler) => window.removeEventListener('popstate', handler)); + popstateHandlers = []; + vi.restoreAllMocks(); vi.unstubAllGlobals(); }); @@ -155,7 +170,7 @@ describe('installSpaAuctionHook', () => { expect(fetchStub).not.toHaveBeenCalled(); }); - it('fetches on replaceState and popstate navigation', async () => { + it('fetches on replaceState navigation', async () => { fetchStub.mockResolvedValue({ ok: true, json: async () => ({ slots: [], bids: {} }), @@ -168,15 +183,44 @@ describe('installSpaAuctionHook', () => { '/__ts/page-bids?path=%2Freplaced', expect.objectContaining({ credentials: 'include' }) ); + }); + it('fetches on popstate navigation to a new path', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + + // Browsers change the URL out-of-band on back/forward, then fire popstate. + // Use the unwrapped history method so the patched handler is not invoked. + originalReplaceState({}, '', '/popped'); window.dispatchEvent(new PopStateEvent('popstate')); await flushAsync(); - expect(fetchStub).toHaveBeenLastCalledWith( - '/__ts/page-bids?path=%2Freplaced', + expect(fetchStub).toHaveBeenCalledWith( + '/__ts/page-bids?path=%2Fpopped', expect.objectContaining({ credentials: 'include' }) ); }); + it('does not re-fetch on popstate to the same path', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + + history.replaceState({}, '', '/replaced'); + await flushAsync(); + expect(fetchStub).toHaveBeenCalledTimes(1); + + // popstate on the same path (hash-only change or scroll-restoration + // back/forward) must not re-request impressions. + window.dispatchEvent(new PopStateEvent('popstate')); + await flushAsync(); + expect(fetchStub).toHaveBeenCalledTimes(1); + }); + it('drops a stale response that resolves after a newer navigation started', async () => { let resolveFirst: ((value: unknown) => void) | undefined; fetchStub diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index f52986c8a..8b5776298 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -3,18 +3,12 @@ // in the build context, so `dead_code` is expected. #![allow(clippy::unwrap_used, clippy::panic, dead_code)] -// Stub out dependencies for build.rs context -mod glob { - pub struct Pattern; - impl Pattern { - pub fn new(_: &str) -> Result { - Ok(Pattern) - } - pub fn matches(&self, _: &str) -> bool { - false - } - } -} +// `glob` is a real build-dependency (see Cargo.toml `[build-dependencies]`), so +// `creative_slot_build_check::pattern_compiles` resolves `glob::Pattern::new` +// against the actual glob crate. It must NOT be stubbed here: a stub that always +// returned `Ok` would let an invalid env-injected pattern such as +// `page_patterns = ["["]` pass the build-time check and embed into the config, +// only to be dropped by the real glob crate at runtime settings load. #[path = "src/error.rs"] mod error; diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index c654d39ee..4b901ee20 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -890,9 +890,16 @@ impl AuctionOrchestrator { break; } }; - remaining = select_result.remaining; + // Destructure so transport failures can be attributed to a provider + // via `failed_backend_name`, mirroring run_providers_parallel. + let crate::platform::PlatformSelectResult { + ready, + remaining: new_remaining, + failed_backend_name, + } = select_result; + remaining = new_remaining; - match select_result.ready { + match ready { Ok(platform_response) => { let backend_name = platform_response.backend_name.clone().unwrap_or_default(); if let Some((provider_name, start_time, provider)) = @@ -920,8 +927,14 @@ impl AuctionOrchestrator { } Err(e) => { log::warn!("Provider '{}' parse failed: {:?}", provider_name, e); - responses - .push(AuctionResponse::error(&provider_name, response_time_ms)); + // Mirror the parallel path so a parse failure is + // attributed (error_type + message) in provider_details. + responses.push(provider_error_response( + &provider_name, + response_time_ms, + ERROR_TYPE_PARSE_RESPONSE, + &e, + )); } } } else { @@ -932,7 +945,32 @@ impl AuctionOrchestrator { } } Err(e) => { - log::warn!("A provider request failed during collection: {:?}", e); + // Mirror the parallel path: attribute the transport failure to + // the provider behind `failed_backend_name` so it appears in + // provider_details instead of vanishing. + if let Some(ref backend_name) = failed_backend_name { + if let Some((provider_name, start_time, _)) = + backend_to_provider.remove(backend_name) + { + let response_time_ms = start_time.elapsed().as_millis() as u64; + log::warn!("Provider '{}' request failed: {:?}", provider_name, e); + responses.push(provider_transport_failed_response( + &provider_name, + response_time_ms, + )); + } else { + log::warn!( + "A provider request failed (backend '{}' not tracked): {:?}", + backend_name, + e + ); + } + } else { + log::warn!( + "A provider request failed during collection (backend not identified): {:?}", + e + ); + } } } diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index c55d98bc0..3090898fb 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -222,9 +222,22 @@ impl CreativeOpportunitySlot { .page_patterns .iter() .filter_map(|pattern| { - Pattern::new(pattern) - .or_else(|_| Pattern::new(&pattern.replace("**", "*"))) - .ok() + match Pattern::new(pattern).or_else(|_| Pattern::new(&pattern.replace("**", "*"))) { + Ok(compiled) => Some(compiled), + Err(_) => { + // Build-time validation only requires *one* valid pattern + // per slot, so a mixed valid/invalid set passes the build + // with the bad pattern silently dropped here. Warn so the + // operator can see the slot matches fewer pages than + // configured. + log::warn!( + "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", + self.id, + pattern + ); + None + } + } }) .collect(); } @@ -370,6 +383,11 @@ pub struct PrebidSlotParams { /// /// Leave empty (or omit `bidders` in config) to auto-expand all /// `config.bidders` with zone-aware param overrides. + /// + /// Note: when this map is non-empty it is forwarded verbatim, so a slot's + /// `targeting.zone` is **not** injected for these bidders (the `trustedServer` + /// expansion key that carries it is only added when `bidders` is empty). Set + /// explicit per-bidder params only when you do not need zone-aware overrides. #[serde(default)] pub bidders: HashMap, } diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index 6a0f446b7..aa2892f9a 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -36,6 +36,48 @@ const ALLOWED_SLOT_FIELDS: &[&str] = &[ "providers", ]; +/// Fields the runtime [`CreativeOpportunityFormat`] accepts. +/// +/// Mirrors the struct's `#[serde(deny_unknown_fields)]`; the build path +/// deserializes formats as raw JSON, so a typo like `mediatype` (for +/// `media_type`) would otherwise embed and fail runtime settings load. +/// +/// [`CreativeOpportunityFormat`]: crate::creative_opportunities::CreativeOpportunityFormat +const ALLOWED_FORMAT_FIELDS: &[&str] = &["width", "height", "media_type"]; + +/// Provider keys the runtime [`SlotProviders`] accepts. +/// +/// [`SlotProviders`]: crate::creative_opportunities::SlotProviders +const ALLOWED_PROVIDER_FIELDS: &[&str] = &["aps", "prebid"]; + +/// Fields the runtime [`ApsSlotParams`] accepts. +/// +/// [`ApsSlotParams`]: crate::creative_opportunities::ApsSlotParams +const ALLOWED_APS_FIELDS: &[&str] = &["slot_id"]; + +/// Fields the runtime [`PrebidSlotParams`] accepts. +/// +/// [`PrebidSlotParams`]: crate::creative_opportunities::PrebidSlotParams +const ALLOWED_PREBID_FIELDS: &[&str] = &["bidders"]; + +/// Rejects any key in `object` that is not in `allowed`, mirroring the runtime +/// struct's `#[serde(deny_unknown_fields)]`. +/// +/// `context` names the offending object in the error (e.g. `` slot `atf` +/// format ``) so a build failure points at the exact config location. +fn reject_unknown_keys( + object: &serde_json::Map, + allowed: &[&str], + context: &str, +) -> Result<(), String> { + for key in object.keys() { + if !allowed.contains(&key.as_str()) { + return Err(format!("{context} has unknown field '{key}'")); + } + } + Ok(()) +} + /// Validate that `value` is a `price_granularity` the runtime can deserialize. /// /// The build context types `price_granularity` as a `String`, so an invalid @@ -114,12 +156,60 @@ pub(crate) fn validate_creative_slot( // `#[serde(deny_unknown_fields)]`. The raw-JSON build path would otherwise // accept env-injected typos that the runtime rejects at settings load. if let Some(object) = slot.as_object() { - for key in object.keys() { - if !ALLOWED_SLOT_FIELDS.contains(&key.as_str()) { - return Err(format!("slot `{id}` has unknown field '{key}'")); + reject_unknown_keys(object, ALLOWED_SLOT_FIELDS, &format!("slot `{id}`"))?; + } + + // Reject nested unknown/mistyped fields too. The runtime's typed structs are + // all `#[serde(deny_unknown_fields)]`, but the raw-JSON build path bypasses + // those checks, so a config like `formats=[{width,height,mediatype}]` or + // `providers={aps={slotId}}` would otherwise pass the build and fail runtime + // settings load. + if let Some(formats) = slot.get("formats").and_then(serde_json::Value::as_array) { + for format in formats { + if let Some(object) = format.as_object() { + reject_unknown_keys( + object, + ALLOWED_FORMAT_FIELDS, + &format!("slot `{id}` format"), + )?; } } } + if let Some(providers) = slot.get("providers").and_then(serde_json::Value::as_object) { + reject_unknown_keys( + providers, + ALLOWED_PROVIDER_FIELDS, + &format!("slot `{id}` providers"), + )?; + if let Some(aps) = providers.get("aps").and_then(serde_json::Value::as_object) { + reject_unknown_keys( + aps, + ALLOWED_APS_FIELDS, + &format!("slot `{id}` providers.aps"), + )?; + } + if let Some(prebid) = providers + .get("prebid") + .and_then(serde_json::Value::as_object) + { + reject_unknown_keys( + prebid, + ALLOWED_PREBID_FIELDS, + &format!("slot `{id}` providers.prebid"), + )?; + } + } + + // An explicit empty/whitespace `div_id` override is rejected, mirroring + // `CreativeOpportunitySlot::validate_runtime`: the injected JS resolves slots + // with `candidate.id.startsWith(slot.div_id)`, and every element id starts + // with the empty string, so an empty override would bind the slot to the + // first id-bearing element in the document. + if let Some(div_id) = slot.get("div_id").and_then(serde_json::Value::as_str) { + if div_id.trim().is_empty() { + return Err(format!("slot `{id}` div_id override must not be empty")); + } + } // At least one page pattern that is non-empty and compiles as a glob. // Runtime preparation drops uncompilable patterns and rejects the slot when @@ -343,6 +433,96 @@ mod tests { assert!(err.contains("GAM unit path"), "got: {err}"); } + #[test] + fn rejects_blank_div_id_override() { + // An empty div_id override binds the slot to the first id-bearing + // element at runtime, so validate_runtime rejects it — the build must + // too, or a CI-green config fails settings load on the deployed service. + let slot = json!({ + "id": "atf", + "div_id": " ", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("blank div_id override must fail at build time"); + assert!( + err.contains("div_id override must not be empty"), + "got: {err}" + ); + } + + #[test] + fn rejects_unknown_format_field() { + // `mediatype` is a typo for `media_type`; the runtime format struct is + // deny_unknown_fields, so the build must reject it. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250, "mediatype": "banner" }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("unknown format field must fail at build time"); + assert!(err.contains("unknown field 'mediatype'"), "got: {err}"); + } + + #[test] + fn rejects_unknown_provider_field() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "providers": { "appnexus": {} } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("unknown provider field must fail at build time"); + assert!(err.contains("unknown field 'appnexus'"), "got: {err}"); + } + + #[test] + fn rejects_unknown_aps_field() { + // `slotId` is a typo for `slot_id`. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "providers": { "aps": { "slotId": "abc" } } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("unknown aps field must fail at build time"); + assert!(err.contains("unknown field 'slotId'"), "got: {err}"); + } + + #[test] + fn rejects_unknown_prebid_field() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "providers": { "prebid": { "bidder": {} } } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("unknown prebid field must fail at build time"); + assert!(err.contains("unknown field 'bidder'"), "got: {err}"); + } + + #[test] + fn accepts_well_formed_nested_provider_config() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250, "media_type": "banner" }], + "providers": { + "aps": { "slot_id": "abc" }, + "prebid": { "bidders": {} } + } + }); + assert!( + validate_creative_slot(&slot, "123456789").is_ok(), + "well-formed nested provider config must be accepted" + ); + } + #[test] fn rejects_missing_id() { let slot = json!({ "page_patterns": ["/"], "formats": [{ "width": 1, "height": 1 }] }); diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index e24a138d4..efaac43fd 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -477,9 +477,12 @@ impl IntegrationHeadInjector for GptIntegration { /// Prebid Universal Creative request proves the TS creative rendered. /// It does **not** trigger refresh auctions or handle GPT slot refresh events. /// - /// Post-`window.load`, slim-Prebid takes over: it listens for GPT refresh - /// events, runs client-side auctions, and sets targeting for subsequent - /// impressions. SPA pushState navigation is also slim-Prebid's domain. + /// Post-`window.load`, slim-Prebid owns scroll and GPT refresh: it listens + /// for GPT refresh events, runs client-side auctions, and sets targeting for + /// subsequent impressions. SPA navigation is handled separately by + /// `installSpaAuctionHook()` in the GPT bundle, which re-runs the server-side + /// auction via `GET /__ts/page-bids` on pushState / replaceState / popstate + /// route changes (see `auction/endpoints.rs`). /// The `POST /auction` endpoint is not involved in scroll or refresh flows. fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { let mut scripts = vec![ @@ -490,9 +493,14 @@ impl IntegrationHeadInjector for GptIntegration { ]; if let Some(ref url) = self.config.slim_prebid_url { + // JSON-encode the URL, then escape `` cannot close this inline tag and + // let trailing markup execute (standard JSON-in-HTML mitigation). + let encoded = serde_json::to_string(url) + .expect("should serialize string") + .replace("window.__tsjs_slim_prebid_url={};", - serde_json::to_string(url).expect("should serialize string") + "" )); } @@ -1298,6 +1306,44 @@ mod tests { ); } + #[test] + fn head_inserts_escapes_script_terminator_in_slim_prebid_url() { + // A configured URL containing `` must not close the inline tag. + let config = GptConfig { + slim_prebid_url: Some("https://cdn.example.com/x".to_string()), + ..test_config() + }; + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + + let inserts = integration.head_inserts(&ctx); + + // The injected `` must be neutralised: the only + // `` left is the tag's own legitimate closer. + assert!( + !inserts[2].contains(" terminator, got: {}", + inserts[2] + ); + assert_eq!( + inserts[2].matches("").count(), + 1, + "only the tag's own closing should remain, got: {}", + inserts[2] + ); + assert!( + inserts[2].contains("<\\/script>"), + "should emit the escaped terminator, got: {}", + inserts[2] + ); + } + #[test] fn head_inserts_omits_slim_prebid_url_when_not_configured() { let integration = GptIntegration::new(test_config()); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index f1bd9d833..e069f3481 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -113,9 +113,10 @@ // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts s.setTargeting("ts_initial", "1"); // Map both the inner div and the GPT slot's element ID (the - // "-container" div when TS defined the slot there) so slotRenderEnded - // — which reports the GPT slot element ID — can find the slot for - // nurl/burl beacon firing. + // "-container" div when TS defined the slot there) into divToSlotId. + // This bootstrap fires no beacons and registers no slotRenderEnded + // listener; the map is consumed by the bundle's render bridge (index.ts) + // once it loads, which reports the GPT slot element ID. divToSlotId[actualDivId] = slot.id; var slotElementId = s.getSlotElementId(); if (slotElementId && slotElementId !== actualDivId) { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 44172a985..4b3979ed9 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1036,12 +1036,6 @@ pub struct AuctionDispatch<'a> { /// /// Returns a [`TrustedServerError`] if the proxy request fails or the /// origin backend is unreachable. -/// -/// # Panics -/// -/// Panics if `should_run_auction` is `true` but `settings.creative_opportunities` is `None`. -/// This is a logic invariant: `should_run_auction` is only set when creative opportunities -/// are configured, so this state is unreachable in practice. pub async fn handle_publisher_request( settings: &Settings, integration_registry: &IntegrationRegistry, @@ -1679,11 +1673,6 @@ fn is_supported_content_encoding(encoding: &str) -> bool { matches!(encoding, "" | "identity" | "gzip" | "deflate" | "br") } -/// Normalizes the client-supplied `path` query parameter before glob matching. -/// -/// The SPA hook sends `location.pathname`, but the parameter is -/// client-controlled: strip any query string or fragment and force a leading -/// `/` so slot `page_patterns` always match against a canonical path shape. /// Same-origin gate for `/__ts/page-bids`. /// /// The endpoint is a side-effecting GET: it dispatches real PBS/APS auctions @@ -1713,6 +1702,11 @@ fn page_bids_request_allowed(req: &Request) -> bool { } } +/// Normalizes the client-supplied `path` query parameter before glob matching. +/// +/// The SPA hook sends `location.pathname`, but the parameter is +/// client-controlled: strip any query string or fragment and force a leading +/// `/` so slot `page_patterns` always match against a canonical path shape. fn normalize_page_bids_path(raw: &str) -> String { let path = raw.split(['?', '#']).next().unwrap_or(""); if path.starts_with('/') { From dc2b18c3ce193fc52719ff06822cca85afe9f59f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 23 Jun 2026 18:33:50 +0530 Subject: [PATCH 119/395] Ignore leftover artifacts in pre-rename crate dirs The EdgeZero sync (#761) renamed crates/js and crates/integration-tests to crates/trusted-server-*. The old directories still hold local-only build artifacts (node_modules, target, dist) whose gitignore rules moved with the rename, so git now sees them as untracked. Ignore the defunct paths until the directories are removed from disk. --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 25e2fa11f..9c6f49e76 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,9 @@ src/*.html /crates/trusted-server-integration-tests/browser/test-results/ /crates/trusted-server-integration-tests/browser/playwright-report/ /crates/trusted-server-integration-tests/browser/.browser-test-state.json + +# Defunct pre-rename crate dirs (renamed to crates/trusted-server-*); ignore the +# leftover local build artifacts (node_modules, target, dist) that remain on disk. +/crates/js/ +/crates/integration-tests/ + From 7d34bbbafed5bb0a3aa43e06e2e8774516cb2818 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 23 Jun 2026 18:34:18 +0530 Subject: [PATCH 120/395] Address PR #680 review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1 — EdgeZero finalize cache/Set-Cookie privacy parity: Share the protected finalizer between the legacy and EdgeZero paths. apply_finalize_headers now strips surrogate cache headers and downgrades cookie-bearing responses to private, and skips operator response_headers that would re-enable shared caching on uncacheable responses; finalize_response delegates to it. The EdgeZero entry point re-applies an HttpResponse enforce_set_cookie_cache_privacy after ec_finalize_response and request-filter effects so a late EC Set-Cookie cannot reach a shared cache. Adds middleware tests for both cases. P1 — empty page-bids must not enable GPT services: adInit() only enables GPT services when it has a slot to display or refresh, and the SPA hook skips adInit() for an empty page-bids response unless prior TS state needs sweeping. Prevents a consent-denied or kill-switched navigation from activating the publisher's GPT setup. P2 — scope Prebid refresh targeting to the refreshed slots: setTargetingForGPTAsync is called with the synthetic refresh ad-unit codes so a one-slot refresh no longer mutates unrelated GPT slots. P2 — validate nested slot value shapes at build time: The creative-slot build check now validates media_type against the runtime MediaType variants, targeting as a string map, page_patterns as strings, providers.aps.slot_id as a string, providers.prebid.bidders as a map, and floor_price as a number — closing build-green/runtime-broken gaps. A drift-guard test ties media_type to the runtime enum. CI — suppress CodeQL cleartext-logging false positives: Annotate the provider/mediator "not registered" warnings; they log static config identifiers, not secrets. --- .../trusted-server-adapter-fastly/src/main.rs | 98 +------ .../src/middleware.rs | 218 +++++++++++++++- .../src/auction/orchestrator.rs | 4 + .../src/creative_slot_build_check.rs | 246 ++++++++++++++++++ .../lib/src/integrations/gpt/index.ts | 26 +- .../lib/src/integrations/prebid/index.ts | 7 +- .../lib/test/integrations/gpt/ad_init.test.ts | 34 +++ .../test/integrations/gpt/spa_hook.test.ts | 43 +++ .../test/integrations/prebid/index.test.ts | 51 ++++ 9 files changed, 636 insertions(+), 91 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 274b37c22..711aa2008 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -6,8 +6,7 @@ use edgezero_core::app::Hooks as _; use edgezero_core::body::Body as EdgeBody; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::http::{ - header, HeaderName, HeaderValue, Method, Request as HttpRequest, Response as HttpResponse, - StatusCode, + header, HeaderValue, Method, Request as HttpRequest, Response as HttpResponse, StatusCode, }; use error_stack::Report; use fastly::http::Method as FastlyMethod; @@ -16,10 +15,7 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::AuctionOrchestrator; use trusted_server_core::auth::enforce_basic_auth; -use trusted_server_core::constants::{ - COOKIE_SHAREDID, COOKIE_TS_EIDS, ENV_FASTLY_IS_STAGING, ENV_FASTLY_SERVICE_VERSION, - HEADER_X_GEO_INFO_AVAILABLE, HEADER_X_TS_ENV, HEADER_X_TS_VERSION, -}; +use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; @@ -374,6 +370,11 @@ fn edgezero_main(mut req: FastlyRequest, config_store: ConfigStoreHandle) { if let Some(effects) = &request_filter_effects { effects.apply_to_response(&mut response); } + // Final cache guard: EC finalization and request-filter + // effects above may have added a per-user Set-Cookie after + // `apply_finalize_headers` ran, so re-apply the privacy + // downgrade before send, mirroring legacy_main. + crate::middleware::enforce_set_cookie_cache_privacy(&mut response); compat::to_fastly_response(response).send_to_client(); if ec_state.is_real_browser { @@ -403,6 +404,9 @@ fn edgezero_main(mut req: FastlyRequest, config_store: ConfigStoreHandle) { if let Some(effects) = &request_filter_effects { effects.apply_to_response(&mut response); } + // Final cache guard for the no-EC-finalization fallback: request-filter + // effects may still have added a per-user Set-Cookie after finalize headers. + crate::middleware::enforce_set_cookie_cache_privacy(&mut response); compat::to_fastly_response(response).send_to_client(); } @@ -1212,84 +1216,10 @@ fn publisher_response_carries_body(method: &Method, status: StatusCode) -> bool /// version/staging, then operator-configured `settings.response_headers`. /// This means operators can intentionally override any managed header. fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: &mut HttpResponse) { - if let Some(geo) = geo_info { - geo.set_response_headers(response); - } else { - response.headers_mut().insert( - HEADER_X_GEO_INFO_AVAILABLE, - HeaderValue::from_static("false"), - ); - } - - if let Ok(v) = ::std::env::var(ENV_FASTLY_SERVICE_VERSION) { - if let Ok(value) = HeaderValue::from_str(&v) { - response.headers_mut().insert(HEADER_X_TS_VERSION, value); - } else { - log::warn!("Skipping invalid FASTLY_SERVICE_VERSION response header value"); - } - } - if ::std::env::var(ENV_FASTLY_IS_STAGING).as_deref() == Ok("1") { - response - .headers_mut() - .insert(HEADER_X_TS_ENV, HeaderValue::from_static("staging")); - } - - // Any response that sets a per-user cookie (notably the EC identity cookie - // minted on a visitor's first navigation) must never be shared-cached, or a - // shared cache could replay one user's Set-Cookie to others. The publisher - // path only forces `private` for HTML that carries inline ad data, so this - // net covers ordinary navigations whose sole per-user payload is the cookie. - // Skip when the response is already uncacheable so we don't clobber a - // stricter directive (e.g. `no-store`). - // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match - // against a lowercased copy — `No-Store` / `Private` must count. - if response.headers().contains_key(header::SET_COOKIE) { - // Surrogate cache headers must come off every cookie-bearing response, - // even one already carrying a stricter `no-store`/`private` directive — - // they are independent of Cache-Control and would otherwise let a shared - // cache store and replay one visitor's Set-Cookie. - response.headers_mut().remove("surrogate-control"); - response.headers_mut().remove("fastly-surrogate-control"); - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); - if !already_uncacheable { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), - ); - } - } - - // Per-user responses (assembled HTML, page-bids, cookie-bearing navigations) - // carry an uncacheable Cache-Control directive (`private` or `no-store`). - // Operator headers must not re-enable shared caching for them — neither by - // replacing Cache-Control nor by reintroducing the surrogate cache headers - // the privacy paths stripped. - let response_is_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); - - for (key, value) in &settings.response_headers { - if response_is_uncacheable - && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) - || key.eq_ignore_ascii_case("surrogate-control") - || key.eq_ignore_ascii_case("fastly-surrogate-control")) - { - continue; - } - let header_name = HeaderName::from_bytes(key.as_bytes()) - .expect("settings.response_headers validated at load time"); - let header_value = - HeaderValue::from_str(value).expect("settings.response_headers validated at load time"); - response.headers_mut().insert(header_name, header_value); - } + // Legacy and EdgeZero paths share one protected finalizer so the cache / + // Set-Cookie privacy hardening cannot drift between them. `HttpResponse` and + // the middleware's `Response` are the same `edgezero_core::http::Response`. + apply_finalize_headers(settings, geo_info, response); } /// Forces cookie-bearing Fastly responses to stay private to shared caches. diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index ceb470b7d..7c24d2dbb 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -16,7 +16,7 @@ use async_trait::async_trait; use edgezero_adapter_fastly::context::FastlyRequestContext; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response, StatusCode}; +use edgezero_core::http::{header, HeaderName, HeaderValue, Response, StatusCode}; use edgezero_core::middleware::{Middleware, Next}; use edgezero_core::response::IntoResponse; use std::net::IpAddr; @@ -181,13 +181,19 @@ where /// Applies all standard Trusted Server response headers to the given response. /// /// Mirrors [`crate::finalize_response`] exactly, operating on [`Response`] from -/// `edgezero_core::http` instead of `HttpResponse`. +/// `edgezero_core::http` instead of `HttpResponse`. [`crate::finalize_response`] +/// delegates here so the legacy and `EdgeZero` paths share one protected +/// finalizer. /// /// Header write order (last write wins): /// 1. Geo headers (`x-geo-*`) — or `X-Geo-Info-Available: false` when absent /// 2. `X-TS-Version` from `FASTLY_SERVICE_VERSION` env var /// 3. `X-TS-ENV: staging` when `FASTLY_IS_STAGING == "1"` -/// 4. `settings.response_headers` — operator-configured overrides applied last +/// 4. Set-Cookie cache privacy — strip surrogate cache headers and downgrade +/// `Cache-Control` to `private, max-age=0` on cookie-bearing responses +/// 5. `settings.response_headers` — operator-configured overrides, except the +/// cache-controlling headers are skipped on uncacheable (`private`/`no-store`) +/// responses so operators cannot re-enable shared caching for per-user payloads pub(crate) fn apply_finalize_headers( settings: &Settings, geo_info: Option<&GeoInfo>, @@ -216,7 +222,32 @@ pub(crate) fn apply_finalize_headers( .insert(HEADER_X_TS_ENV, HeaderValue::from_static("staging")); } + // Any response that sets a per-user cookie (notably the EC identity cookie) + // must never be shared-cached, or a shared cache could replay one user's + // Set-Cookie to others. Skip when the response is already uncacheable so we + // don't clobber a stricter directive (e.g. `no-store`). + enforce_set_cookie_cache_privacy(response); + + // Per-user responses (assembled HTML, page-bids, cookie-bearing navigations) + // carry an uncacheable Cache-Control directive (`private` or `no-store`). + // Operator headers must not re-enable shared caching for them — neither by + // replacing Cache-Control nor by reintroducing the surrogate cache headers + // the privacy paths stripped. + let response_is_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")); + for (key, value) in &settings.response_headers { + if response_is_uncacheable + && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) + || key.eq_ignore_ascii_case("surrogate-control") + || key.eq_ignore_ascii_case("fastly-surrogate-control")) + { + continue; + } let header_name = HeaderName::from_bytes(key.as_bytes()) .expect("should be a valid header name: response_headers validated in prepare_runtime"); let header_value = HeaderValue::from_str(value).expect( @@ -226,6 +257,44 @@ pub(crate) fn apply_finalize_headers( } } +/// Forces cookie-bearing responses to stay private to shared caches. +/// +/// Mirrors [`crate::enforce_set_cookie_cache_privacy`] for the [`Response`] type +/// from `edgezero_core::http`. The `EdgeZero` entry point re-applies this after +/// [`ec_finalize_response`](trusted_server_core::ec::finalize::ec_finalize_response) +/// and request-filter effects, because the EC identity `Set-Cookie` is written +/// after [`apply_finalize_headers`] runs and would otherwise reach a shared cache +/// with inherited `public`/surrogate cache headers. +/// +/// Idempotent: a response already marked `private`/`no-store` keeps its stricter +/// `Cache-Control`, but the surrogate cache headers are stripped regardless so a +/// `no-store` cookie response can never retain shared cacheability. +pub(crate) fn enforce_set_cookie_cache_privacy(response: &mut Response) { + if !response.headers().contains_key(header::SET_COOKIE) { + return; + } + // Surrogate cache headers must come off every cookie-bearing response, even + // one already carrying a stricter `no-store`/`private` directive — they are + // independent of Cache-Control and would otherwise let a shared cache store + // and replay one visitor's Set-Cookie. + response.headers_mut().remove("surrogate-control"); + response.headers_mut().remove("fastly-surrogate-control"); + // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match + // against a lowercased copy — `No-Store` / `Private` must count. + let already_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")); + if !already_uncacheable { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=0"), + ); + } +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -342,6 +411,149 @@ mod tests { ); } + fn response_with_headers(pairs: &[(&'static str, &'static str)]) -> Response { + let mut response = empty_response(); + for (key, value) in pairs { + response.headers_mut().insert( + HeaderName::from_static(key), + HeaderValue::from_static(value), + ); + } + response + } + + #[test] + fn apply_finalize_headers_downgrades_public_set_cookie_response() { + // A per-user cookie response that arrives shared-cacheable (origin-public + // plus a surrogate directive) must be downgraded so a shared cache cannot + // store and replay one visitor's Set-Cookie. + let settings = settings_with_response_headers(vec![]); + let mut response = response_with_headers(&[ + ("set-cookie", "ts-ec=abc; Path=/"), + ("cache-control", "public, max-age=600"), + ("surrogate-control", "max-age=600"), + ]); + + apply_finalize_headers(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "should downgrade a public cookie response to private" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip surrogate-control from a cookie-bearing response" + ); + } + + #[test] + fn apply_finalize_headers_blocks_operator_surrogate_on_private_response() { + // Operator response_headers must not re-enable shared caching for an + // uncacheable (private) per-user response — neither by replacing + // Cache-Control nor by reintroducing surrogate cache headers. + let settings = settings_with_response_headers(vec![ + ("cache-control", "public, max-age=3600"), + ("surrogate-control", "max-age=3600"), + ("x-operator", "kept"), + ]); + let mut response = response_with_headers(&[("cache-control", "private, max-age=0")]); + + apply_finalize_headers(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "operator cache-control must not weaken a private response" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "operator surrogate-control must not be applied to a private response" + ); + assert_eq!( + response + .headers() + .get("x-operator") + .and_then(|v| v.to_str().ok()), + Some("kept"), + "non-cache operator headers must still apply" + ); + } + + #[test] + fn enforce_set_cookie_cache_privacy_downgrades_late_cookie() { + // Mirrors the EdgeZero post-ec_finalize guard: a Set-Cookie added after + // finalize headers ran (origin-public response) must be downgraded. + let mut response = response_with_headers(&[ + ("set-cookie", "ts-ec=abc; Path=/"), + ("cache-control", "public, max-age=600"), + ("surrogate-control", "max-age=600"), + ]); + + enforce_set_cookie_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "should downgrade a late public cookie response to private" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip surrogate-control from the late cookie response" + ); + } + + #[test] + fn enforce_set_cookie_cache_privacy_keeps_stricter_no_store() { + // Idempotent: a stricter no-store directive is preserved, but surrogate + // headers still come off. + let mut response = response_with_headers(&[ + ("set-cookie", "ts-ec=abc; Path=/"), + ("cache-control", "no-store"), + ("surrogate-control", "max-age=600"), + ]); + + enforce_set_cookie_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store"), + "should keep the stricter no-store directive" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip surrogate-control even when keeping no-store" + ); + } + + #[test] + fn enforce_set_cookie_cache_privacy_ignores_cookieless_response() { + let mut response = response_with_headers(&[("cache-control", "public, max-age=600")]); + + enforce_set_cookie_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("public, max-age=600"), + "should leave a cookieless response untouched" + ); + } + // --------------------------------------------------------------------------- // FinalizeResponseMiddleware::handle tests // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 36ae00c58..18efd1865 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -748,6 +748,8 @@ impl AuctionOrchestrator { let provider = match self.providers.get(provider_name) { Some(p) => p, None => { + // lgtm[rust/cleartext-logging] + // The provider name is a static config identifier (e.g. "prebid"), not a secret. log::warn!("Provider '{}' not registered, skipping", provider_name); continue; } @@ -1108,6 +1110,8 @@ impl AuctionOrchestrator { } } None => { + // lgtm[rust/cleartext-logging] + // The mediator name is a static config identifier, not a secret. log::warn!("Mediator '{}' not registered", mediator_name); (None, self.select_winning_bids(&responses, &floor_prices)) } diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index aa2892f9a..15e5ca98a 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -104,6 +104,57 @@ pub(crate) fn validate_price_granularity(value: &str) -> Result<(), String> { }) } +/// Accepted `media_type` values, mirroring the runtime [`MediaType`] enum's +/// `#[serde(rename_all = "lowercase")]` variants. +/// +/// The build path types a format's `media_type` as raw JSON, so a value such as +/// `"bannerr"` would embed cleanly and then fail runtime settings load — the real +/// [`MediaType`] enum cannot deserialize it. A crate-context test +/// (`media_type_values_match_runtime_enum`) asserts this list stays in lockstep +/// with the enum's `Deserialize` impl, so the two cannot drift. +/// +/// [`MediaType`]: crate::auction::types::MediaType +const MEDIA_TYPE_VALUES: &[&str] = &["banner", "video", "native"]; + +/// Validate a format's `media_type` value against the runtime [`MediaType`] enum. +/// +/// # Errors +/// +/// Returns an error string when `value` is not a JSON string naming one of the +/// runtime [`MediaType`] variants. +/// +/// [`MediaType`]: crate::auction::types::MediaType +fn validate_media_type(value: &serde_json::Value, slot_id: &str) -> Result<(), String> { + let media_type = value + .as_str() + .ok_or_else(|| format!("slot `{slot_id}` format media_type must be a string"))?; + if !MEDIA_TYPE_VALUES.contains(&media_type) { + return Err(format!( + "slot `{slot_id}` format media_type '{media_type}' is invalid; expected one of: banner, video, native" + )); + } + Ok(()) +} + +/// Validate that `value` is a string→string map, mirroring a runtime +/// `HashMap` field. +/// +/// # Errors +/// +/// Returns an error string when `value` is not a JSON object or any of its values +/// is not a JSON string. `context` names the offending field in the error. +fn validate_string_map(value: &serde_json::Value, context: &str) -> Result<(), String> { + let object = value + .as_object() + .ok_or_else(|| format!("{context} must be a map of string keys to string values"))?; + for (key, entry) in object { + if !entry.is_string() { + return Err(format!("{context} value for '{key}' must be a string")); + } + } + Ok(()) +} + /// Returns `true` when `id` is non-empty and only `[A-Za-z0-9_-]`. fn is_valid_slot_id(id: &str) -> bool { !id.is_empty() @@ -172,6 +223,12 @@ pub(crate) fn validate_creative_slot( ALLOWED_FORMAT_FIELDS, &format!("slot `{id}` format"), )?; + // Validate the nested `media_type` value, not just the field + // name: a value like `"bannerr"` passes the key check but the + // runtime `MediaType` enum cannot deserialize it. + if let Some(media_type) = object.get("media_type") { + validate_media_type(media_type, id)?; + } } } } @@ -187,6 +244,15 @@ pub(crate) fn validate_creative_slot( ALLOWED_APS_FIELDS, &format!("slot `{id}` providers.aps"), )?; + // `ApsSlotParams::slot_id` is a `String`; a non-string value embeds + // cleanly but fails runtime deserialization. + if let Some(slot_id_value) = aps.get("slot_id") { + if !slot_id_value.is_string() { + return Err(format!( + "slot `{id}` providers.aps.slot_id must be a string" + )); + } + } } if let Some(prebid) = providers .get("prebid") @@ -197,6 +263,29 @@ pub(crate) fn validate_creative_slot( ALLOWED_PREBID_FIELDS, &format!("slot `{id}` providers.prebid"), )?; + // `PrebidSlotParams::bidders` is a map; a non-object value (e.g. a + // bare string or array) fails runtime deserialization. + if let Some(bidders) = prebid.get("bidders") { + if !bidders.is_object() { + return Err(format!( + "slot `{id}` providers.prebid.bidders must be a map of bidder names to params" + )); + } + } + } + } + + // `targeting` is a runtime `HashMap`; a non-string value + // (e.g. `targeting = { pos = 1 }`) embeds cleanly but fails settings load. + if let Some(targeting) = slot.get("targeting") { + validate_string_map(targeting, &format!("slot `{id}` targeting"))?; + } + + // `floor_price` is an `Option`; a non-numeric value would fail the + // runtime deserialization the build path otherwise bypasses. + if let Some(floor_price) = slot.get("floor_price") { + if !floor_price.is_null() && floor_price.as_f64().is_none() { + return Err(format!("slot `{id}` floor_price must be a number")); } } @@ -211,6 +300,18 @@ pub(crate) fn validate_creative_slot( } } + // `page_patterns` is a runtime `Vec`; a non-string entry (e.g. + // `page_patterns = [123]`) fails deserialization. The validity check below + // skips non-strings via `filter_map`, so reject them explicitly first. + if let Some(patterns) = slot + .get("page_patterns") + .and_then(serde_json::Value::as_array) + { + if patterns.iter().any(|p| !p.is_string()) { + return Err(format!("slot `{id}` page_patterns entries must be strings")); + } + } + // At least one page pattern that is non-empty and compiles as a glob. // Runtime preparation drops uncompilable patterns and rejects the slot when // none remain, so a private/env config like `page_patterns = ["["]` would @@ -523,6 +624,151 @@ mod tests { ); } + #[test] + fn rejects_invalid_media_type() { + // `bannerr` passes the field-name check but the runtime MediaType enum + // cannot deserialize it, so settings load would fail on the service. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250, "media_type": "bannerr" }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("invalid media_type must fail at build time"); + assert!( + err.contains("media_type 'bannerr' is invalid"), + "got: {err}" + ); + } + + #[test] + fn rejects_non_string_media_type() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250, "media_type": 1 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-string media_type must fail at build time"); + assert!(err.contains("media_type must be a string"), "got: {err}"); + } + + #[test] + fn accepts_all_media_types() { + for media_type in ["banner", "video", "native"] { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250, "media_type": media_type }] + }); + assert!( + validate_creative_slot(&slot, "123456789").is_ok(), + "'{media_type}' should be a valid media_type" + ); + } + } + + #[test] + fn media_type_values_match_runtime_enum() { + use crate::auction::types::MediaType; + // Every listed value must deserialize into the runtime enum. + for value in super::MEDIA_TYPE_VALUES { + serde_json::from_value::(json!(value)) + .unwrap_or_else(|_| panic!("'{value}' should deserialize into MediaType")); + } + // Exhaustive match so a newly added MediaType variant forces this test + // (and MEDIA_TYPE_VALUES) to be updated, preventing silent drift. + for variant in [MediaType::Banner, MediaType::Video, MediaType::Native] { + let covered = match variant { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + }; + assert!( + super::MEDIA_TYPE_VALUES.contains(&covered), + "MEDIA_TYPE_VALUES is missing runtime variant '{covered}'" + ); + } + } + + #[test] + fn rejects_non_string_targeting_value() { + // `targeting` is a runtime HashMap; a numeric value + // embeds cleanly but fails settings load. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "targeting": { "pos": 1 } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-string targeting value must fail at build time"); + assert!( + err.contains("targeting value for 'pos' must be a string"), + "got: {err}" + ); + } + + #[test] + fn rejects_non_string_aps_slot_id() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "providers": { "aps": { "slot_id": 123 } } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-string aps slot_id must fail at build time"); + assert!( + err.contains("providers.aps.slot_id must be a string"), + "got: {err}" + ); + } + + #[test] + fn rejects_non_object_prebid_bidders() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "providers": { "prebid": { "bidders": "appnexus" } } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-object prebid bidders must fail at build time"); + assert!( + err.contains("providers.prebid.bidders must be a map"), + "got: {err}" + ); + } + + #[test] + fn rejects_non_numeric_floor_price() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "floor_price": "high" + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-numeric floor_price must fail at build time"); + assert!(err.contains("floor_price must be a number"), "got: {err}"); + } + + #[test] + fn rejects_non_string_page_pattern_entry() { + let slot = json!({ + "id": "atf", + "page_patterns": [123], + "formats": [{ "width": 300, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-string page_patterns entry must fail at build time"); + assert!( + err.contains("page_patterns entries must be strings"), + "got: {err}" + ); + } + #[test] fn rejects_missing_id() { let slot = json!({ "page_patterns": ["/"], "formats": [{ "width": 1, "height": 1 }] }); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 30071220d..73b9419c6 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -536,8 +536,18 @@ export function installTsAdInit(): void { ts.divToSlotId = divToSlotId; ts.prevSlotTargetingKeys = nextSlotTargetingKeys; - // enableSingleRequest and enableServices must only be called once per page load. - if (!ts.servicesEnabled) { + // Whether this call produced any TS slot to render. A gated page-bids + // response (auction kill switch or consent denial) returns no slots, so + // the loops above leave these empty. + const hasRenderableWork = slotsToDisplay.length > 0 || slotsToRefresh.length > 0; + + // enableSingleRequest and enableServices must only be called once per page + // load. Skip activating GPT services when TS has nothing to display or + // refresh and has not already enabled them: a consent-denied or + // kill-switched navigation must not turn on the publisher's GPT services + // or race their own setup. The targeting sweep above still runs so stale + // TS targeting from a prior navigation is cleared. + if (!ts.servicesEnabled && hasRenderableWork) { g.pubads!().enableSingleRequest(); g.enableServices?.(); ts.servicesEnabled = true; @@ -693,7 +703,17 @@ export function installSpaAuctionHook(): void { if (inflight !== controller) return; ts.adSlots = data.slots; ts.bids = data.bids; - ts.adInit?.(); + // An empty page-bids response (auction kill switch or consent gate) carries + // no TS slots. Only run adInit() when there are slots to apply or prior TS + // state to sweep — otherwise a consent-denied or kill-switched navigation + // must not enter the GPT command queue and risk activating services. + const hasPriorTsState = + (ts.prevGptSlots?.length ?? 0) > 0 || + Object.keys(ts.prevSlotTargetingKeys ?? {}).length > 0 || + Object.keys(ts.divToSlotId ?? {}).length > 0; + if (data.slots.length > 0 || hasPriorTsState) { + ts.adInit?.(); + } } catch (err) { if (err instanceof DOMException && err.name === 'AbortError') return; log.warn('SPA auction hook: fetch failed', err); diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index fe175b44d..40a8d9e2e 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -779,10 +779,15 @@ export function installRefreshHandler(timeoutMs = 1500): void { }; }); + // Scope GPT targeting to just the synthetic refresh ad units. An unscoped + // call would set hb_* targeting on every ad unit with known bids, mutating + // unrelated GPT slots whose targeting this wrapper only cleared for + // `targetSlots` — leaving their next request dependent on stale state. + const refreshAdUnitCodes = adUnits.map((unit) => unit.code); pbjs.requestBids({ adUnits, bidsBackHandler: () => { - pbjs.setTargetingForGPTAsync?.(); + pbjs.setTargetingForGPTAsync?.(refreshAdUnitCodes); originalRefresh(targetSlots, opts); }, timeout: timeoutMs, diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index d649778bc..b82542695 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -337,6 +337,40 @@ describe('installTsAdInit', () => { expect((window as TestWindow).tsjs!.prevSlotTargetingKeys).toEqual({}); }); + it('does not enable GPT services when the page-bids response has no slots', async () => { + // A gated page-bids response returns no slots. With nothing to display or + // refresh and services not already enabled, adInit() must not call + // enableSingleRequest()/enableServices() and activate the publisher's GPT + // services on a consent-denied or kill-switched navigation. + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([]), + addEventListener: vi.fn(), + refresh: vi.fn(), + }; + const enableServices = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn(), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices, + }; + (window as TestWindow).tsjs = { + adSlots: [], + bids: {}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(mockPubads.enableSingleRequest).not.toHaveBeenCalled(); + expect(enableServices).not.toHaveBeenCalled(); + expect((window as TestWindow).tsjs!.servicesEnabled).toBeFalsy(); + expect(mockPubads.refresh).not.toHaveBeenCalled(); + }); + it('keeps the GAM path when debug adm is present', async () => { const slotEl = document.getElementById('div-atf-sidebar')!; const mockSlot = { diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 6be0a8484..28854a902 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -89,6 +89,49 @@ describe('installSpaAuctionHook', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('skips adInit on an empty page-bids response with no prior TS state', async () => { + // A gated page-bids response (auction kill switch or consent denial) returns + // no slots. With no prior TS state to sweep, the hook must not call adInit() + // so a consent-denied navigation cannot activate the publisher's GPT setup. + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/gated-route'); + await flushAsync(); + + expect(ts.adSlots).toEqual([]); + expect(ts.bids).toEqual({}); + expect(adInit).not.toHaveBeenCalled(); + }); + + it('runs adInit on an empty page-bids response when prior TS state exists', async () => { + // When TS touched slots on a previous navigation, an empty response still + // needs adInit() to sweep the stale TS targeting from those slots. + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + ts.prevSlotTargetingKeys = { 'div-prev': ['hb_pb'] }; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/cleanup-route'); + await flushAsync(); + + expect(ts.adSlots).toEqual([]); + expect(adInit).toHaveBeenCalledTimes(1); + }); + it('defers applying bids until the route ad container is inserted', async () => { fetchStub.mockResolvedValue({ ok: true, diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index fd7703546..738a1cc78 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -864,6 +864,57 @@ describe('prebid/installRefreshHandler', () => { ); }); + it('scopes the GPT targeting call to the refreshed slot code', () => { + const setTargetingForGPTAsync = vi.fn(); + (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; + // Run the bidsBackHandler synchronously so the targeting call fires. + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const originalRefresh = vi.fn(); + // Only the header slot is refreshed; the footer slot must be untouched. + const headerSlot = { + getSlotElementId: vi.fn(() => 'div-ad-header'), + getTargeting: vi.fn(() => []), + clearTargeting: vi.fn().mockReturnThis(), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [headerSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'header_ad', + gam_unit_path: '/123/header', + div_id: 'div-ad-header', + formats: [[728, 90]], + targeting: { zone: 'header' }, + }, + { + id: 'footer_ad', + gam_unit_path: '/123/footer', + div_id: 'div-ad-footer', + formats: [[728, 90]], + targeting: { zone: 'footer' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh([headerSlot]); + + expect(setTargetingForGPTAsync).toHaveBeenCalledTimes(1); + expect(setTargetingForGPTAsync).toHaveBeenCalledWith(['div-ad-header']); + expect(originalRefresh).toHaveBeenCalledWith([headerSlot], undefined); + + delete (mockPbjs as any).setTargetingForGPTAsync; + }); + it('includes configured client-side bidders in refresh ad units', () => { (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; // Original publisher ad unit carries a client-side rubicon bid. From 5a835c13a137e8850f82a71df426a4d88a82c768 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 23 Jun 2026 18:40:26 +0530 Subject: [PATCH 121/395] Add glob to the integration-tests lockfile The merge took main's trusted-server-integration-tests Cargo.lock, but the branch's trusted-server-core now pulls in glob (the creative-slot build check uses glob::Pattern). The integration crate path-depends on core, so its locked graph was missing glob and the --locked CI build refused to update it. Add only glob v0.3.3; no other versions change, keeping the shared direct-dependency parity check green. --- crates/trusted-server-integration-tests/Cargo.lock | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-integration-tests/Cargo.lock b/crates/trusted-server-integration-tests/Cargo.lock index 48d0af29e..692858beb 100644 --- a/crates/trusted-server-integration-tests/Cargo.lock +++ b/crates/trusted-server-integration-tests/Cargo.lock @@ -1502,6 +1502,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "group" version = "0.13.0" @@ -3755,7 +3761,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4154,6 +4160,7 @@ dependencies = [ "fastly", "flate2", "futures", + "glob", "hex", "hmac", "http", @@ -4169,6 +4176,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "subtle", + "tokio", "toml", "trusted-server-js", "trusted-server-openrtb", From 975b4aa0dbb578d92735616dba2d45900c42b062 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 25 Jun 2026 20:25:21 +0530 Subject: [PATCH 122/395] Fix server-side ad template review blockers --- .../src/backend.rs | 82 +++++++++++++++---- .../trusted-server-adapter-fastly/src/main.rs | 74 ++++++++++++++++- .../src/platform.rs | 14 +++- .../src/auction/orchestrator.rs | 38 +++++---- .../trusted-server-core/src/ec/pull_sync.rs | 1 + .../src/integrations/datadome/protection.rs | 1 + .../src/integrations/mod.rs | 1 + .../src/integrations/prebid.rs | 68 ++++++++++++--- .../src/platform/test_support.rs | 1 + .../trusted-server-core/src/platform/types.rs | 2 + crates/trusted-server-core/src/proxy.rs | 2 + crates/trusted-server-core/src/publisher.rs | 1 + 12 files changed, 232 insertions(+), 53 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/backend.rs b/crates/trusted-server-adapter-fastly/src/backend.rs index 7763eaf0e..4056c81da 100644 --- a/crates/trusted-server-adapter-fastly/src/backend.rs +++ b/crates/trusted-server-adapter-fastly/src/backend.rs @@ -49,6 +49,8 @@ fn sanitize_backend_name_component(value: &str) -> String { /// Default first-byte timeout for backends (15 seconds). pub(crate) const DEFAULT_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +/// Default timeout between response body bytes for backends (10 seconds). +pub(crate) const DEFAULT_BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(10); /// Configuration for creating a dynamic Fastly backend. /// @@ -60,6 +62,7 @@ pub struct BackendConfig<'a> { port: Option, certificate_check: bool, first_byte_timeout: Duration, + between_bytes_timeout: Duration, host_header_override: Option<&'a str>, } @@ -76,6 +79,7 @@ impl<'a> BackendConfig<'a> { port: None, certificate_check: true, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_BETWEEN_BYTES_TIMEOUT, host_header_override: None, } } @@ -106,6 +110,17 @@ impl<'a> BackendConfig<'a> { self } + /// Set the maximum time to wait between response body bytes. + /// + /// Defaults to 10 seconds. Auction backends should set this to the same + /// remaining budget as the first-byte timeout so slow-drip bodies cannot + /// hold the auction past its deadline. + #[must_use] + pub fn between_bytes_timeout(mut self, timeout: Duration) -> Self { + self.between_bytes_timeout = timeout; + self + } + /// Set the outbound Host header sent to the backend origin. #[must_use] pub fn host_header_override(mut self, host: Option<&'a str>) -> Self { @@ -159,13 +174,15 @@ impl<'a> BackendConfig<'a> { } else { "_nocert" }; - let timeout_ms = self.first_byte_timeout.as_millis(); + let first_byte_timeout_ms = self.first_byte_timeout.as_millis(); + let between_bytes_timeout_ms = self.between_bytes_timeout.as_millis(); let backend_name = format!( - "backend_{}{}{}_t{}", + "backend_{}{}{}_fb{}_bb{}", sanitize_backend_name_component(&name_base), host_override_suffix, cert_suffix, - timeout_ms + first_byte_timeout_ms, + between_bytes_timeout_ms ); Ok((backend_name, target_port)) @@ -187,9 +204,10 @@ impl<'a> BackendConfig<'a> { /// Ensure a dynamic backend exists for this configuration and return its name. /// /// The backend name is derived from the scheme, host, port, certificate - /// setting, and `first_byte_timeout` to avoid collisions. Different - /// timeout values produce different backend registrations so that a - /// tight deadline cannot be silently widened by an earlier registration. + /// setting, `first_byte_timeout`, and `between_bytes_timeout` to avoid + /// collisions. Different timeout values produce different backend + /// registrations so that a tight deadline cannot be silently widened by an + /// earlier registration. /// /// # Errors /// @@ -210,7 +228,7 @@ impl<'a> BackendConfig<'a> { .override_host(&host_header) .connect_timeout(Duration::from_secs(1)) .first_byte_timeout(self.first_byte_timeout) - .between_bytes_timeout(Duration::from_secs(10)); + .between_bytes_timeout(self.between_bytes_timeout); if self.scheme.eq_ignore_ascii_case("https") { builder = builder.enable_ssl().sni_hostname(self.host); if self.certificate_check { @@ -381,7 +399,7 @@ mod tests { let name = BackendConfig::new("https", "origin.example.com") .ensure() .expect("should create backend for valid HTTPS origin"); - assert_eq!(name, "backend_https_origin_example_com_443_t15000"); + assert_eq!(name, "backend_https_origin_example_com_443_fb15000_bb10000"); } #[test] @@ -390,7 +408,10 @@ mod tests { .certificate_check(false) .ensure() .expect("should create backend with cert check disabled"); - assert_eq!(name, "backend_https_origin_example_com_443_nocert_t15000"); + assert_eq!( + name, + "backend_https_origin_example_com_443_nocert_fb15000_bb10000" + ); } #[test] @@ -399,7 +420,7 @@ mod tests { .port(Some(8080)) .ensure() .expect("should create backend for HTTP origin with explicit port"); - assert_eq!(name, "backend_http_api_test-site_org_8080_t15000"); + assert_eq!(name, "backend_http_api_test-site_org_8080_fb15000_bb10000"); } #[test] @@ -407,7 +428,7 @@ mod tests { let name = BackendConfig::new("http", "example.org") .ensure() .expect("should create backend defaulting to port 80 for HTTP"); - assert_eq!(name, "backend_http_example_org_80_t15000"); + assert_eq!(name, "backend_http_example_org_80_fb15000_bb10000"); } #[test] @@ -464,11 +485,11 @@ mod tests { ); assert_eq!( name_a, - "backend_https_origin_example_com_443_oh_www_example_com_t15000" + "backend_https_origin_example_com_443_oh_www_example_com_fb15000_bb10000" ); assert_eq!( name_b, - "backend_https_origin_example_com_443_oh_m_example_com_t15000" + "backend_https_origin_example_com_443_oh_m_example_com_fb15000_bb10000" ); } @@ -523,12 +544,39 @@ mod tests { "backends with different timeouts should have different names" ); assert!( - name_a.ends_with("_t2000"), - "name should include timeout suffix" + name_a.ends_with("_fb2000_bb10000"), + "name should include first-byte and between-bytes timeout suffix" + ); + assert!( + name_b.ends_with("_fb500_bb10000"), + "name should include first-byte and between-bytes timeout suffix" + ); + } + + #[test] + fn different_between_bytes_timeouts_produce_different_names() { + use std::time::Duration; + + let (name_a, _) = BackendConfig::new("https", "origin.example.com") + .between_bytes_timeout(Duration::from_secs(2)) + .compute_name() + .expect("should compute name with 2000ms between-bytes timeout"); + let (name_b, _) = BackendConfig::new("https", "origin.example.com") + .between_bytes_timeout(Duration::from_millis(500)) + .compute_name() + .expect("should compute name with 500ms between-bytes timeout"); + + assert_ne!( + name_a, name_b, + "backends with different between-bytes timeouts should have different names" + ); + assert!( + name_a.ends_with("_fb15000_bb2000"), + "name should include first-byte and between-bytes timeout suffix" ); assert!( - name_b.ends_with("_t500"), - "name should include timeout suffix" + name_b.ends_with("_fb15000_bb500"), + "name should include first-byte and between-bytes timeout suffix" ); } } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 711aa2008..017f1a9ef 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -143,6 +143,10 @@ fn is_edgezero_enabled(config_store: &ConfigStoreHandle) -> Result bool { + settings.creative_opportunities.is_none() +} + fn health_response(req: &FastlyRequest) -> Option { if req.get_method() == FastlyMethod::GET && req.get_path() == "/health" { return Some(FastlyResponse::from_status(200).with_body_text_plain("ok")); @@ -194,8 +198,24 @@ fn main() { log::warn!("failed to read edgezero_enabled flag, falling back to legacy path: {e}"); false }) { - log::debug!("routing request through EdgeZero path"); - edgezero_main(req, edgezero_config_store); + match get_settings() { + Ok(settings) if edgezero_can_handle_settings(&settings) => { + log::debug!("routing request through EdgeZero path"); + edgezero_main(req, edgezero_config_store); + } + Ok(_) => { + log::warn!( + "EdgeZero path does not yet support creative_opportunities; routing through legacy path" + ); + legacy_main(req); + } + Err(e) => { + log::warn!( + "failed to load settings for EdgeZero compatibility check, falling back to legacy path: {e:?}" + ); + legacy_main(req); + } + } } else { log::debug!("routing request through legacy path"); legacy_main(req); @@ -1340,6 +1360,36 @@ mod tests { .expect("should parse test settings") } + fn test_settings_with_creative_opportunities() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + + [creative_opportunities] + gam_network_id = "12345" + auction_timeout_ms = 500 + "#, + ) + .expect("should parse test settings with creative opportunities") + } + #[test] fn parses_true_flag_values() { assert!(parse_edgezero_flag("true"), "should parse 'true'"); @@ -1367,6 +1417,26 @@ mod tests { assert!(!parse_edgezero_flag("yes"), "should not parse 'yes'"); } + #[test] + fn edgezero_accepts_settings_without_creative_opportunities() { + let settings = test_settings(); + + assert!( + edgezero_can_handle_settings(&settings), + "should allow EdgeZero when server-side ad templates are not configured" + ); + } + + #[test] + fn edgezero_rejects_settings_with_creative_opportunities() { + let settings = test_settings_with_creative_opportunities(); + + assert!( + !edgezero_can_handle_settings(&settings), + "should route through legacy path while EdgeZero lacks server-side ad-template support" + ); + } + #[test] fn health_response_short_circuits_get_health() { let req = FastlyRequest::get("https://example.com/health"); diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 9b1a73422..935c957ea 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -159,6 +159,7 @@ fn backend_config_from_spec(spec: &PlatformBackendSpec) -> BackendConfig<'_> { .host_header_override(spec.host_header_override.as_deref()) .certificate_check(spec.certificate_check) .first_byte_timeout(spec.first_byte_timeout) + .between_bytes_timeout(spec.between_bytes_timeout) } impl PlatformBackend for FastlyPlatformBackend { @@ -676,6 +677,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let name = backend @@ -683,7 +685,7 @@ mod tests { .expect("should compute backend name for valid spec"); assert_eq!( - name, "backend_https_origin_example_com_443_t15000", + name, "backend_https_origin_example_com_443_fb15000_bb15000", "should match BackendConfig naming convention" ); } @@ -698,6 +700,7 @@ mod tests { host_header_override: Some("www.example.com".to_string()), certificate_check: true, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let name = backend @@ -705,7 +708,7 @@ mod tests { .expect("should compute backend name for host header override"); assert_eq!( - name, "backend_https_origin_example_com_443_oh_www_example_com_t15000", + name, "backend_https_origin_example_com_443_oh_www_example_com_fb15000_bb15000", "should match BackendConfig naming convention with host header override" ); } @@ -720,6 +723,7 @@ mod tests { host_header_override: None, certificate_check: false, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let name = backend @@ -742,6 +746,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let result = backend.predict_name(&spec); @@ -759,6 +764,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_millis(2000), + between_bytes_timeout: Duration::from_millis(2000), }; let name = backend @@ -766,8 +772,8 @@ mod tests { .expect("should compute name with custom timeout"); assert!( - name.ends_with("_t2000"), - "should encode 2000ms timeout in name" + name.ends_with("_fb2000_bb2000"), + "should encode 2000ms first-byte and between-bytes timeouts in name" ); } diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 18efd1865..53656568a 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -389,9 +389,9 @@ impl AuctionOrchestrator { } // Give each provider only the remaining time from the auction - // deadline so that its backend first_byte_timeout doesn't extend - // past the overall budget. Also respect the provider's own - // configured timeout when it is tighter than the remaining budget. + // deadline so that backend transport timeouts do not extend past + // the overall budget. Also respect the provider's own configured + // timeout when it is tighter than the remaining budget. let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); let effective_timeout = remaining_ms.min(provider.timeout_ms()); @@ -488,10 +488,11 @@ impl AuctionOrchestrator { // Enforce the auction deadline: after each select() returns, check // elapsed time and drop remaining requests if the timeout is exceeded. // - // NOTE: `select()` blocks until at least one backend responds (or its - // transport timeout fires). Hard deadline enforcement therefore depends - // on every backend's `first_byte_timeout` being set to at most the - // remaining auction budget — which Phase 1 above guarantees. + // NOTE: `select()` blocks until at least one backend responds and, on + // some adapters, buffers the selected response body before returning. + // Hard deadline enforcement therefore depends on every backend's + // first-byte and between-bytes timeouts being set to at most the + // remaining auction budget, which Phase 1 above guarantees. let mut remaining = pending_requests; while !remaining.is_empty() { @@ -976,12 +977,13 @@ impl AuctionOrchestrator { } } - // Drain every dispatched request. Each backend was capped with a - // first-byte timeout at dispatch time, so by the collect phase the - // remaining handles may already be ready even if wall-clock time - // elapsed while the origin was slow — dropping them here would - // discard SSP responses that already arrived. The mediator launch - // below still observes A_deadline via `remaining_budget_ms`. + // Drain every dispatched request. Each backend was capped with + // first-byte and between-bytes timeouts at dispatch time, so by the + // collect phase the remaining handles may already be ready even if + // wall-clock time elapsed while the origin was slow. Dropping them + // here would discard SSP responses that already arrived. The + // mediator launch below still observes A_deadline via + // `remaining_budget_ms`. } let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { @@ -990,11 +992,11 @@ impl AuctionOrchestrator { // Cap the mediator at whichever is tighter: its own configured // timeout or the remaining auction budget (A_deadline). The old // comment here claimed origin drain could exhaust the budget before - // collection, but SSP backends are given first_byte_timeout = - // effective_timeout (capped at their provider timeout) at dispatch - // time, so they cannot run past A_deadline independently. Giving - // the mediator an uncapped timeout lets it run past A_deadline, - // violating the bounded hold invariant. + // collection, but SSP backends are given first-byte and between-bytes + // timeouts equal to effective_timeout (capped at their provider + // timeout) at dispatch time, so they cannot run past A_deadline + // independently. Giving the mediator an uncapped timeout lets it run + // past A_deadline, violating the bounded hold invariant. let remaining = remaining_budget_ms(auction_start, timeout_ms); if remaining == 0 { log::warn!( diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index fbc64f776..fa096d59d 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -173,6 +173,7 @@ pub fn dispatch_pull_sync( host_header_override: None, certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, }) { Ok(name) => name, Err(err) => { diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 4ae15c927..717ad46e8 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -159,6 +159,7 @@ impl DataDomeIntegration { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_millis(u64::from(self.config.timeout_ms)), + between_bytes_timeout: Duration::from_millis(u64::from(self.config.timeout_ms)), }; services.backend().ensure(&spec).change_context(Self::error( diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 3678f949c..7777b79d4 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -152,6 +152,7 @@ fn integration_backend_spec( host_header_override: None, certificate_check, first_byte_timeout, + between_bytes_timeout: first_byte_timeout, }) } diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index f81268724..5aa7827a7 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1080,13 +1080,13 @@ impl PrebidAuctionProvider { // Build user object — populate consent at both OpenRTB 2.6 top-level // and Prebid ext-based locations (dual placement). - // In cookies_only mode, body consent fields are omitted — consent - // travels exclusively through the forwarded Cookie header. - let consent_ctx = if self.config.consent_forwarding.includes_body_consent() { - request.user.consent.as_ref() - } else { - None - }; + // In cookies_only mode, cookie-sourced consent travels through the + // forwarded Cookie header. KV/policy-sourced consent has no inbound + // cookie to forward, so carry it in the OpenRTB body instead. + let consent_ctx = request.user.consent.as_ref().filter(|ctx| { + self.config.consent_forwarding.includes_body_consent() + || !matches!(ctx.source, crate::consent::ConsentSource::Cookie) + }); let raw_tc = consent_ctx.and_then(|c| c.raw_tc_string.clone()); let user = Some(User { id: request.user.id.clone(), @@ -1809,7 +1809,7 @@ mod tests { AdFormat, AdSlot, AuctionContext, AuctionRequest, DeviceInfo, PublisherInfo, UserInfo, }; - use crate::consent::ConsentContext; + use crate::consent::{ConsentContext, ConsentSource}; use crate::geo::GeoInfo; use crate::html_processor::{create_html_processor, HtmlProcessorConfig}; use crate::integrations::{ @@ -1863,10 +1863,11 @@ mod tests { spec: &PlatformBackendSpec, ) -> Result> { Ok(format!( - "predicted_{}_{}_{}", + "predicted_{}_{}_{}_{}", spec.scheme, spec.host, - spec.first_byte_timeout.as_millis() + spec.first_byte_timeout.as_millis(), + spec.between_bytes_timeout.as_millis() )) } @@ -1902,8 +1903,8 @@ mod tests { .expect("should predict backend name through platform backend"); assert_eq!( - backend_name, "predicted_https_prebid.example_123", - "should use PlatformBackend::predict_name instead of duplicating the naming scheme" + backend_name, "predicted_https_prebid.example_123_123", + "should cap both first-byte and between-bytes timeouts to the auction budget" ); } @@ -2713,6 +2714,49 @@ server_url = "https://prebid.example" ); } + #[test] + fn to_openrtb_includes_kv_consent_when_cookies_only_has_no_cookie_to_forward() { + let mut config = base_config(); + config.consent_forwarding = ConsentForwardingMode::CookiesOnly; + let provider = PrebidAuctionProvider::new(config); + let mut auction_request = create_test_auction_request(); + auction_request.user.consent = Some(ConsentContext { + raw_tc_string: Some("BOkv-backed-consent-string".to_string()), + raw_us_privacy: Some("1YNN".to_string()), + gdpr_applies: true, + source: ConsentSource::KvStore, + ..Default::default() + }); + + let settings = make_settings(); + let request = build_test_request(); + assert!( + !request.headers().contains_key(header::COOKIE), + "test request should not carry a consent cookie to forward" + ); + let context = create_test_auction_context(&settings, &request); + + let openrtb = provider.to_openrtb( + &auction_request, + &context, + None, + make_request_info(&context), + ); + + assert_eq!( + openrtb.user.as_ref().and_then(|u| u.consent.as_deref()), + Some("BOkv-backed-consent-string"), + "cookies_only should fall back to body consent when consent came from KV" + ); + let regs = openrtb.regs.as_ref().expect("should include consent regs"); + assert_eq!(regs.gdpr, Some(true), "should carry GDPR applicability"); + assert_eq!( + regs.us_privacy.as_deref(), + Some("1YNN"), + "should carry non-cookie consent strings from KV" + ); + } + #[test] fn to_openrtb_sets_gdpr_true_for_non_eu_country_with_consent() { // When geo says non-GDPR but a consent string is present, the consent diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 0b14afe65..d744060cf 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -808,6 +808,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, }; let name = stub.ensure(&spec).expect("should return a backend name"); assert_eq!(name, "stub-backend", "should return fixed name"); diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 77f7d6c5e..23f57a580 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -139,6 +139,8 @@ pub struct PlatformBackendSpec { pub certificate_check: bool, /// Maximum time to wait for the first response byte. pub first_byte_timeout: Duration, + /// Maximum time to wait between response body bytes. + pub between_bytes_timeout: Duration, } /// Cloneable container of platform services for a single request. diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index f8f6af4ca..8e9072d48 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -1073,6 +1073,7 @@ pub async fn handle_asset_proxy_request( host_header_override: None, certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, }) .change_context(TrustedServerError::Proxy { message: "asset backend registration failed".to_string(), @@ -1256,6 +1257,7 @@ async fn proxy_with_redirects( host_header_override: None, certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, }) .change_context(TrustedServerError::Proxy { message: "backend registration failed".to_string(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 95c3cfeaa..2f79114ed 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1150,6 +1150,7 @@ pub async fn handle_publisher_request( host_header_override: settings.publisher.origin_host_header_override.clone(), certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT, }) .change_context(TrustedServerError::Proxy { message: "backend registration failed".to_string(), From ae17b45a3fc7d237c14797b06bfbafe9e608ad11 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 25 Jun 2026 22:13:11 +0530 Subject: [PATCH 123/395] Fix EdgeZero empty ad-template config gate --- .../trusted-server-adapter-fastly/src/main.rs | 58 +++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 017f1a9ef..423e4d1f5 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -144,7 +144,10 @@ fn is_edgezero_enabled(config_store: &ConfigStoreHandle) -> Result bool { - settings.creative_opportunities.is_none() + settings + .creative_opportunities + .as_ref() + .is_none_or(|creative_opportunities| creative_opportunities.slot.is_empty()) } fn health_response(req: &FastlyRequest) -> Option { @@ -205,7 +208,7 @@ fn main() { } Ok(_) => { log::warn!( - "EdgeZero path does not yet support creative_opportunities; routing through legacy path" + "EdgeZero path does not yet support configured creative_opportunity slots; routing through legacy path" ); legacy_main(req); } @@ -1360,7 +1363,7 @@ mod tests { .expect("should parse test settings") } - fn test_settings_with_creative_opportunities() -> Settings { + fn test_settings_with_empty_creative_opportunities() -> Settings { Settings::from_toml( r#" [[handlers]] @@ -1390,6 +1393,41 @@ mod tests { .expect("should parse test settings with creative opportunities") } + fn test_settings_with_configured_creative_opportunities() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + + [creative_opportunities] + gam_network_id = "12345" + auction_timeout_ms = 500 + + [[creative_opportunities.slot]] + id = "atf" + page_patterns = ["/article/*"] + formats = [{ width = 300, height = 250 }] + "#, + ) + .expect("should parse test settings with configured creative opportunities") + } + #[test] fn parses_true_flag_values() { assert!(parse_edgezero_flag("true"), "should parse 'true'"); @@ -1428,8 +1466,18 @@ mod tests { } #[test] - fn edgezero_rejects_settings_with_creative_opportunities() { - let settings = test_settings_with_creative_opportunities(); + fn edgezero_accepts_settings_with_empty_creative_opportunities() { + let settings = test_settings_with_empty_creative_opportunities(); + + assert!( + edgezero_can_handle_settings(&settings), + "should allow EdgeZero when server-side ad templates are configured but no slots are enabled" + ); + } + + #[test] + fn edgezero_rejects_settings_with_configured_creative_opportunity_slots() { + let settings = test_settings_with_configured_creative_opportunities(); assert!( !edgezero_can_handle_settings(&settings), From a11f94689279bb4ba8f686a2cd74572b0341ad3d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 29 Jun 2026 22:56:17 +0530 Subject: [PATCH 124/395] Address fourth-pass PR review findings Blocking: - Move tokio to [dev-dependencies] in trusted-server-core; it was only used by #[tokio::test] and was linking the runtime into the wasm prod build. Confirmed the release wasm adapter build no longer pulls tokio. - Roll back the SPA currentPath on a failed /__ts/page-bids fetch so a transient error no longer permanently strands that route (gpt/index.ts). Build/runtime parity and diagnostics: - Bound creative-opportunity format width/height to u32 range at build time so values the runtime u32 cannot hold are rejected early. - Add #[serde(deny_unknown_fields)] to the build.rs config stub to match the runtime type and reject mistyped table keys at build time. - Warn when the end-tag handler is absent so a silently non-rendering server-side ad feature is diagnosable. - Log dropped slot bidders that are neither configured nor the aps provider. - Log build_bid_index collisions (multiple bids per seat/imp). JS correctness: - Narrow uid.atype to a number before the range check in sanitizeAuctionUid. - Resolve findInjectedSlotForRefresh by exact/container match before the prefix fallback, with a regression test for prefix-overlapping div_ids. - Guard the gpt_bootstrap prefix scan against an empty div_id. - Route injectAdmIntoSlot through findSlotElementByDivId for consistency. Cleanup and docs: - Remove the dead has_post_processors routing dependency from classify_response_route and (now unused) handle_publisher_request. - Extract the duplicated EID resolution/consent-gating/device tail shared by the initial-page and page-bids dispatch paths into one helper. - Anchor the surrogate cache-header list in a shared const so the legacy and EdgeZero Set-Cookie privacy paths stay aligned. - Refresh stale docs (PublisherResponse::Stream, the publisher module platform-coupling note, and UserInfo.eids consent-gate location). --- crates/trusted-server-adapter-axum/src/app.rs | 1 - .../src/app.rs | 1 - .../trusted-server-adapter-fastly/src/app.rs | 1 - .../trusted-server-adapter-fastly/src/main.rs | 6 +- .../src/middleware.rs | 11 +- crates/trusted-server-adapter-spin/src/app.rs | 1 - crates/trusted-server-core/Cargo.toml | 2 +- crates/trusted-server-core/build.rs | 1 + .../trusted-server-core/src/auction/types.rs | 6 +- .../src/creative_slot_build_check.rs | 10 +- .../trusted-server-core/src/html_processor.rs | 9 + .../src/integrations/adserver_mock.rs | 24 +- .../src/integrations/gpt_bootstrap.js | 1 + .../src/integrations/prebid.rs | 11 + crates/trusted-server-core/src/publisher.rs | 262 +++++++++--------- .../lib/src/integrations/gpt/index.ts | 16 +- .../lib/src/integrations/prebid/index.ts | 24 +- .../test/integrations/prebid/index.test.ts | 57 ++++ 18 files changed, 276 insertions(+), 168 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 882aa69c8..8cd53d48f 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -176,7 +176,6 @@ async fn dispatch_fallback( }; handle_publisher_request( &state.settings, - &state.registry, services, None, &mut ec_context, diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 92b1c17e7..d4e8fb3d6 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -305,7 +305,6 @@ fn build_router(state: &Arc) -> RouterService { }; handle_publisher_request( &state.settings, - &state.registry, &services, None, &mut ec_context, diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 8d2d9ae13..13a348314 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -729,7 +729,6 @@ async fn dispatch_fallback( }; handle_publisher_request( &state.settings, - &state.registry, &publisher_services, ec.kv_graph.as_ref(), &mut ec.ec_context, diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index e1c0da9a5..30e91b01f 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -1270,7 +1270,6 @@ async fn route_request( match handle_publisher_request( settings, - integration_registry, runtime_services, kv_graph.as_ref(), &mut ec_context, @@ -1399,8 +1398,9 @@ fn enforce_set_cookie_cache_privacy(response: &mut FastlyResponse) { // keeping a stricter `no-store`/`private` directive — Surrogate-Control is // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. - response.remove_header("surrogate-control"); - response.remove_header("fastly-surrogate-control"); + for name in crate::middleware::SURROGATE_CACHE_HEADERS { + response.remove_header(*name); + } let already_uncacheable = response .get_header_str("cache-control") .map(str::to_ascii_lowercase) diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 6aa9cddd4..91b7dcdec 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -257,6 +257,12 @@ pub(crate) fn apply_finalize_headers( } } +/// Surrogate cache headers stripped from every cookie-bearing response. A single +/// source of truth so the legacy ([`crate::enforce_set_cookie_cache_privacy`]) +/// and `EdgeZero` copies of the privacy downgrade cannot drift apart. +pub(crate) const SURROGATE_CACHE_HEADERS: &[&str] = + &["surrogate-control", "fastly-surrogate-control"]; + /// Forces cookie-bearing responses to stay private to shared caches. /// /// Mirrors [`crate::enforce_set_cookie_cache_privacy`] for the [`Response`] type @@ -277,8 +283,9 @@ pub(crate) fn enforce_set_cookie_cache_privacy(response: &mut Response) { // one already carrying a stricter `no-store`/`private` directive — they are // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. - response.headers_mut().remove("surrogate-control"); - response.headers_mut().remove("fastly-surrogate-control"); + for name in SURROGATE_CACHE_HEADERS { + response.headers_mut().remove(*name); + } // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match // against a lowercased copy — `No-Store` / `Private` must count. let already_uncacheable = response diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index b1b341c17..287bbfd8a 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -607,7 +607,6 @@ fn build_router(state: &Arc) -> RouterService { }; handle_publisher_request( &state.settings, - &state.registry, &services, None, &mut ec_context, diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index cdb280ed1..ab62f53e8 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -40,7 +40,6 @@ serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } subtle = { workspace = true } -tokio = { workspace = true } toml = { workspace = true } trusted-server-js = { path = "../trusted-server-js" } trusted-server-openrtb = { path = "../trusted-server-openrtb" } @@ -83,6 +82,7 @@ test-utils = [] criterion = { workspace = true } edgezero-core = { workspace = true, features = ["test-utils"] } temp-env = { workspace = true } +tokio = { workspace = true } [[bench]] name = "consent_decode" diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index a95c307a1..ef6546285 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -53,6 +53,7 @@ mod creative_opportunities { } #[derive(Debug, Clone, Deserialize, Serialize)] + #[serde(deny_unknown_fields)] pub struct CreativeOpportunitiesConfig { pub gam_network_id: String, #[serde(default)] diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 2a2985926..ffe918aa4 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -87,8 +87,10 @@ pub struct UserInfo { /// Extended User IDs parsed from the [`crate::constants::COOKIE_TS_EIDS`] cookie. /// /// Raw (un-gated) values from the browser; consent gating via - /// [`crate::consent::gate_eids_by_consent`] is applied in the provider - /// layer before any EID reaches a bid request. + /// [`crate::consent::gate_eids_by_consent`] is applied centrally in the + /// endpoint handlers (the auction and page-bids paths) before any EID + /// reaches a bid request — the provider layer just forwards already-gated + /// EIDs. #[serde(skip)] pub eids: Option>, } diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index 15e5ca98a..066cdde1a 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -337,9 +337,15 @@ pub(crate) fn validate_creative_slot( for format in formats { let width = format.get("width").and_then(serde_json::Value::as_u64); let height = format.get("height").and_then(serde_json::Value::as_u64); - if !matches!((width, height), (Some(w), Some(h)) if w > 0 && h > 0) { + // Runtime dimensions are `u32`, so a value above `u32::MAX` passes + // a bare `> 0` check here but fails `from_value::` at runtime + // settings load on every request — the exact failure this build + // check exists to prevent. + let in_u32 = + |v: Option| matches!(v, Some(n) if n > 0 && n <= u64::from(u32::MAX)); + if !(in_u32(width) && in_u32(height)) { return Err(format!( - "slot `{id}` format must have positive width and height" + "slot `{id}` format must have positive width and height within u32 range" )); } } diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 627468adc..a3170084a 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -366,6 +366,15 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso Ok(()) }); handlers.push(handler); + } else { + // No end tag (implicitly closed or EOF ``): lol_html + // cannot attach an end-tag handler, so tsjs.bids/adInit() are + // never injected even though adSlots was injected at ``. + // The whole server-side ad feature then silently fails to + // render — warn so the failure is diagnosable. + log::warn!( + "`` has no end tag (implicitly closed or EOF); tsjs.bids and adInit() were not injected — server-side ads will not render" + ); } Ok(()) } diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 4fff04660..bd3538c71 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -108,14 +108,24 @@ fn build_bid_index(bidder_responses: &[AuctionResponse]) -> BidIndex { let mut index = BidIndex::new(); for response in bidder_responses { for bid in &response.bids { - index.insert( - ( - response.provider.clone(), - bid.slot_id.clone(), - bid.bidder.clone(), - ), - bid.clone(), + let key = ( + response.provider.clone(), + bid.slot_id.clone(), + bid.bidder.clone(), ); + // OpenRTB permits a seat to return multiple bids per imp. This index + // is last-write-wins, so a collision means an earlier bid's + // nurl/burl/cache_* are dropped and win/billing-URL restoration can + // be mis-attributed during mediation. Low severity for the mock + // mediator, but log it so the collision is visible. + if index.insert(key, bid.clone()).is_some() { + log::debug!( + "adserver_mock: duplicate bid for (provider '{}', slot '{}', bidder '{}'); keeping the last — win/billing URL restoration may be mis-attributed", + response.provider, + bid.slot_id, + bid.bidder + ); + } } } index diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index f40283e87..cc4c5c00c 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -68,6 +68,7 @@ for (var i = 0; i < idElements.length; i++) { var candidate = idElements[i]; if ( + slot.div_id && candidate.id.startsWith(slot.div_id) && !candidate.id.endsWith("-container") ) { diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 6e0703d45..451650611 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1017,6 +1017,17 @@ impl PrebidAuctionProvider { bidder.extend(expand_trusted_server_bidders(&self.config.bidders, params)); } else if self.config.bidders.iter().any(|b| b == name) { bidder.insert(name.clone(), params.clone()); + } else if name != "aps" { + // `aps` is intentionally handled by its own provider. Any + // other unrecognized key is likely a misconfiguration (a + // slot bidder absent from `config.bidders`) that silently + // yields an empty bidder map and a stored-request no-bid — + // log it so the drop is diagnosable. + log::debug!( + "prebid: dropping slot '{}' bidder '{}' — not in config.bidders and not a known provider key", + slot.id, + name + ); } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 3ce825c86..b85a5ad22 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -10,21 +10,19 @@ //! streaming processor treats unknown encodings as identity, so publisher code //! must gate them out before the body enters the rewrite pipeline. //! -//! **Note on platform coupling:** This module is currently coupled to -//! `fastly::Body`/`Request`/`Response` at its handler boundaries — the entry -//! points ([`handle_publisher_request`], [`stream_publisher_body`]) still -//! accept and return `fastly::Body` and `fastly::Response`. The streaming -//! processor itself is generic: `process_response_streaming` writes into -//! any [`Write`] (a `Vec` for buffered routes, a `StreamingBody` for the -//! streaming route). The HTTP-type coupling will be addressed in the -//! platform HTTP-type migration alongside all other -//! `fastly::Request`/`Response`/`Body` migrations. It is not a -//! content-rewriting concern. +//! **Note on platform coupling:** The handler boundaries use portable HTTP +//! types: [`handle_publisher_request`] and [`stream_publisher_body`] take and +//! return `http::Request`/`http::Response` over `EdgeBody`, and platform I/O is +//! reached through `RuntimeServices` rather than `fastly::*` directly. The +//! streaming processor itself is generic: `process_response_streaming` writes +//! into any [`Write`] (a `Vec` for buffered routes, a streaming writer for +//! the streaming route). It is not a content-rewriting concern. use std::io::Write; use std::sync::{Arc, Mutex}; use std::time::Duration; +use cookie::CookieJar; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{header, HeaderValue, Method, Request, Response, StatusCode, Uri}; @@ -45,7 +43,7 @@ use crate::ec::EcContext; use crate::error::TrustedServerError; use crate::http_util::{is_navigation_request, serve_static_with_etag, RequestInfo}; use crate::integrations::IntegrationRegistry; -use crate::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; use crate::price_bucket::{price_bucket, PriceGranularity}; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; @@ -333,9 +331,9 @@ pub enum PublisherResponse { Buffered(Response), /// Response headers are ready for a streaming response. Covers processable /// content on any status (2xx or non-2xx — e.g., branded 404/500 HTML and - /// error JSON still get URL rewriting) where the encoding is supported - /// and either the content is non-HTML or no HTML post-processors are - /// registered. The caller must: + /// error JSON still get URL rewriting) where the encoding is supported. + /// Post-processors run inside the streaming processor, so processable HTML + /// is streamed regardless of whether any are registered. The caller must: /// 1. Call `finalize_response()` on the response /// 2. Call `response.stream_to_client()` to get a `StreamingBody` /// 3. Call `stream_publisher_body()` with the body and streaming writer @@ -398,7 +396,6 @@ pub(crate) fn classify_response_route( content_type: &str, content_encoding: &str, request_host: &str, - _has_post_processors: bool, ) -> ResponseRoute { if status == StatusCode::NO_CONTENT || status == StatusCode::RESET_CONTENT { return ResponseRoute::BufferedUnmodified; @@ -1150,7 +1147,6 @@ pub struct AuctionDispatch<'a> { /// origin backend is unreachable. pub async fn handle_publisher_request( settings: &Settings, - integration_registry: &IntegrationRegistry, services: &RuntimeServices, kv: Option<&KvIdentityGraph>, ec_context: &mut EcContext, @@ -1307,33 +1303,19 @@ pub async fn handle_publisher_request( .get("user-agent") .and_then(|v| v.to_str().ok()), ); - let ts_eids_value = cookie_jar - .as_ref() - .and_then(|j| j.get(COOKIE_TS_EIDS)) - .map(|c| c.value().to_owned()); - let client_eids = if ec_id.is_some() { - resolve_client_auction_eids(None, ts_eids_value.as_deref()) - } else { - None - }; - let kv_eids = resolve_auction_eids(kv, auction.registry, ec_context); - let merged_eids = merge_auction_eids(client_eids, kv_eids); - let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); - auction_request.user.eids = - gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); - if had_eids && auction_request.user.eids.is_none() { - log::warn!("Server-side auction EIDs stripped by TCF consent gating"); - } - let client_ip = services.client_info().client_ip.map(|ip| ip.to_string()); - if client_ip.is_some() || geo.is_some() { - let device = auction_request.device.get_or_insert(DeviceInfo { - user_agent: None, - ip: None, - geo: None, - }); - device.ip = client_ip; - device.geo = geo.clone(); - } + apply_auction_eids_and_device( + &mut auction_request, + &AuctionEidTargeting { + cookie_jar: cookie_jar.as_ref(), + ec_id, + kv, + partner_registry: auction.registry, + ec_context, + services, + geo: geo.as_ref(), + path_label: "Server-side", + }, + ); let auction_context = AuctionContext { settings, request: &req, @@ -1439,15 +1421,7 @@ pub async fn handle_publisher_request( .map(|h| h.to_str().unwrap_or_default()) .unwrap_or_default() .to_lowercase(); - let has_post_processors = integration_registry.has_html_post_processors(); - - let route = classify_response_route( - status, - &content_type, - &content_encoding, - request_host, - has_post_processors, - ); + let route = classify_response_route(status, &content_type, &content_encoding, request_host); match route { ResponseRoute::PassThrough => { @@ -1541,6 +1515,70 @@ pub(crate) struct MatchedSlotsContext<'a> { pub request_path: &'a str, } +/// Borrowed inputs for [`apply_auction_eids_and_device`], bundled to keep the +/// helper within the project's 7-argument cap. +struct AuctionEidTargeting<'a> { + cookie_jar: Option<&'a CookieJar>, + ec_id: Option<&'a str>, + kv: Option<&'a KvIdentityGraph>, + partner_registry: Option<&'a PartnerRegistry>, + ec_context: &'a EcContext, + services: &'a RuntimeServices, + geo: Option<&'a GeoInfo>, + /// Prefix for the consent-stripped warning (e.g. `"Server-side"`). + path_label: &'a str, +} + +/// Resolves client + KV EIDs, consent-gates them onto `auction_request`, and +/// attaches the client IP/geo to its device record. +/// +/// Shared verbatim by the initial-page and page-bids dispatch paths so the EID +/// resolution and consent gating live in one place; `path_label` only varies +/// the consent-stripped warning message. +fn apply_auction_eids_and_device( + auction_request: &mut AuctionRequest, + targeting: &AuctionEidTargeting<'_>, +) { + let ts_eids_value = targeting + .cookie_jar + .and_then(|j| j.get(COOKIE_TS_EIDS)) + .map(|c| c.value().to_owned()); + let client_eids = if targeting.ec_id.is_some() { + resolve_client_auction_eids(None, ts_eids_value.as_deref()) + } else { + None + }; + let kv_eids = resolve_auction_eids( + targeting.kv, + targeting.partner_registry, + targeting.ec_context, + ); + let merged_eids = merge_auction_eids(client_eids, kv_eids); + let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); + auction_request.user.eids = + gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); + if had_eids && auction_request.user.eids.is_none() { + log::warn!( + "{} auction EIDs stripped by TCF consent gating", + targeting.path_label + ); + } + let client_ip = targeting + .services + .client_info() + .client_ip + .map(|ip| ip.to_string()); + if client_ip.is_some() || targeting.geo.is_some() { + let device = auction_request.device.get_or_insert(DeviceInfo { + user_agent: None, + ip: None, + geo: None, + }); + device.ip = client_ip; + device.geo = targeting.geo.cloned(); + } +} + /// Build an [`AuctionRequest`] from matched creative opportunity slots. pub(crate) fn build_auction_request( slots_ctx: &MatchedSlotsContext<'_>, @@ -1948,33 +1986,19 @@ pub async fn handle_page_bids( .get("user-agent") .and_then(|v| v.to_str().ok()), ); - let ts_eids_value = cookie_jar - .as_ref() - .and_then(|j| j.get(COOKIE_TS_EIDS)) - .map(|c| c.value().to_owned()); - let client_eids = if ec_id.is_some() { - resolve_client_auction_eids(None, ts_eids_value.as_deref()) - } else { - None - }; - let kv_eids = resolve_auction_eids(kv, auction.registry, ec_context); - let merged_eids = merge_auction_eids(client_eids, kv_eids); - let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); - auction_request.user.eids = - gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); - if had_eids && auction_request.user.eids.is_none() { - log::warn!("Page-bids auction EIDs stripped by TCF consent gating"); - } - let client_ip = services.client_info().client_ip.map(|ip| ip.to_string()); - if client_ip.is_some() || geo.is_some() { - let device = auction_request.device.get_or_insert(DeviceInfo { - user_agent: None, - ip: None, - geo: None, - }); - device.ip = client_ip; - device.geo = geo.clone(); - } + apply_auction_eids_and_device( + &mut auction_request, + &AuctionEidTargeting { + cookie_jar: cookie_jar.as_ref(), + ec_id, + kv, + partner_registry: auction.registry, + ec_context, + services, + geo: geo.as_ref(), + path_label: "Page-bids", + }, + ); let timeout_ms = co_config .auction_timeout_ms .unwrap_or(settings.auction.timeout_ms); @@ -2306,10 +2330,9 @@ mod tests { /// Drive `handle_publisher_request` with no creative opportunities — a plain /// proxy with no server-side auction. Hides the auction/EC wiring so callers - /// read like a simple `(settings, registry, services, req)` proxy. + /// read like a simple `(settings, services, req)` proxy. async fn run_publisher_proxy( settings: &Settings, - integration_registry: &IntegrationRegistry, services: &RuntimeServices, req: Request, ) -> PublisherResponse { @@ -2318,7 +2341,6 @@ mod tests { EcContext::read_from_request(settings, &req, services).expect("should read EC context"); handle_publisher_request( settings, - integration_registry, services, None, &mut ec_context, @@ -2336,8 +2358,6 @@ mod tests { #[tokio::test] async fn publisher_request_uses_platform_http_client_with_http_types() { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"origin response".to_vec()); let services = build_services_with_http_client( @@ -2350,7 +2370,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); - let response = match run_publisher_proxy(&settings, ®istry, &services, req).await { + let response = match run_publisher_proxy(&settings, &services, req).await { PublisherResponse::Buffered(r) => r, PublisherResponse::PassThrough { mut response, body } => { *response.body_mut() = body; @@ -2378,8 +2398,6 @@ mod tests { // exactly the conditions under which the old inline call would have // generated one. let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"ok".to_vec()); let services = build_services_with_http_client( @@ -2408,7 +2426,6 @@ mod tests { let _ = handle_publisher_request( &settings, - ®istry, &services, None, &mut ec_context, @@ -2647,8 +2664,7 @@ mod tests { StatusCode::OK, "text/html; charset=utf-8", "zstd", - "example.com", - false, + "example.com" ), ResponseRoute::BufferedUnmodified, ); @@ -2702,8 +2718,7 @@ mod tests { StatusCode::OK, "text/html; charset=utf-8", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::Stream, ); @@ -2716,8 +2731,7 @@ mod tests { StatusCode::OK, "Text/HTML; Charset=utf-8", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::Stream, "HTML MIME type matching must be case-insensitive", @@ -2731,8 +2745,7 @@ mod tests { StatusCode::OK, "text/html; charset=utf-8", "gzip", - "example.com", - true, + "example.com" ), ResponseRoute::Stream, ); @@ -2741,13 +2754,7 @@ mod tests { #[test] fn route_streams_non_html_even_with_post_processors_registered() { assert_eq!( - classify_response_route( - StatusCode::OK, - "application/json", - "gzip", - "example.com", - true, - ), + classify_response_route(StatusCode::OK, "application/json", "gzip", "example.com"), ResponseRoute::Stream, ); } @@ -2755,7 +2762,7 @@ mod tests { #[test] fn route_buffers_unmodified_on_unsupported_encoding() { assert_eq!( - classify_response_route(StatusCode::OK, "text/html", "zstd", "example.com", false,), + classify_response_route(StatusCode::OK, "text/html", "zstd", "example.com"), ResponseRoute::BufferedUnmodified, ); } @@ -2763,7 +2770,7 @@ mod tests { #[test] fn route_passes_through_non_processable_2xx() { assert_eq!( - classify_response_route(StatusCode::OK, "image/png", "", "example.com", false,), + classify_response_route(StatusCode::OK, "image/png", "", "example.com"), ResponseRoute::PassThrough, ); } @@ -2771,7 +2778,7 @@ mod tests { #[test] fn route_buffers_non_processable_error_responses() { assert_eq!( - classify_response_route(StatusCode::NOT_FOUND, "image/png", "", "example.com", false,), + classify_response_route(StatusCode::NOT_FOUND, "image/png", "", "example.com"), ResponseRoute::BufferedUnmodified, ); } @@ -2779,13 +2786,7 @@ mod tests { #[test] fn route_excludes_204_from_pass_through() { assert_eq!( - classify_response_route( - StatusCode::NO_CONTENT, - "image/png", - "", - "example.com", - false, - ), + classify_response_route(StatusCode::NO_CONTENT, "image/png", "", "example.com"), ResponseRoute::BufferedUnmodified, ); } @@ -2793,13 +2794,7 @@ mod tests { #[test] fn route_excludes_205_from_pass_through() { assert_eq!( - classify_response_route( - StatusCode::RESET_CONTENT, - "image/png", - "", - "example.com", - false, - ), + classify_response_route(StatusCode::RESET_CONTENT, "image/png", "", "example.com"), ResponseRoute::BufferedUnmodified, ); } @@ -2811,8 +2806,7 @@ mod tests { StatusCode::NO_CONTENT, "text/html; charset=utf-8", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::BufferedUnmodified, "204 + HTML must not route to Stream", @@ -2822,8 +2816,7 @@ mod tests { StatusCode::NO_CONTENT, "text/html; charset=utf-8", "gzip", - "example.com", - true, + "example.com" ), ResponseRoute::BufferedUnmodified, "204 + HTML + post-processors must not route to Stream", @@ -2837,8 +2830,7 @@ mod tests { StatusCode::RESET_CONTENT, "application/json", "", - "example.com", - false, + "example.com" ), ResponseRoute::BufferedUnmodified, "205 + JSON must not route to Stream", @@ -2852,8 +2844,7 @@ mod tests { StatusCode::NOT_FOUND, "text/html; charset=utf-8", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::Stream, ); @@ -2862,8 +2853,7 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR, "application/json", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::Stream, ); @@ -2876,8 +2866,7 @@ mod tests { StatusCode::NOT_FOUND, "text/html; charset=utf-8", "gzip", - "example.com", - true, + "example.com" ), ResponseRoute::Stream, ); @@ -2886,7 +2875,7 @@ mod tests { #[test] fn route_passes_through_non_processable_even_with_empty_request_host() { assert_eq!( - classify_response_route(StatusCode::OK, "image/png", "", "", false,), + classify_response_route(StatusCode::OK, "image/png", "", ""), ResponseRoute::PassThrough, ); } @@ -2894,7 +2883,7 @@ mod tests { #[test] fn route_buffers_processable_content_with_empty_request_host() { assert_eq!( - classify_response_route(StatusCode::OK, "text/html", "gzip", "", false,), + classify_response_route(StatusCode::OK, "text/html", "gzip", ""), ResponseRoute::BufferedUnmodified, ); } @@ -3186,8 +3175,6 @@ mod tests { async fn publisher_request_sends_configured_host_header_override() { let mut settings = create_test_settings(); settings.publisher.origin_host_header_override = Some("www.example.com".to_string()); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"origin response".to_vec()); let services = build_services_with_http_client( @@ -3200,7 +3187,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); - let _ = run_publisher_proxy(&settings, ®istry, &services, req).await; + let _ = run_publisher_proxy(&settings, &services, req).await; let recorded_headers = stub.recorded_request_headers(); let outbound_headers = recorded_headers @@ -3465,7 +3452,6 @@ mod tests { "text/html; charset=utf-8", "", "proxy.example.com", - registry.has_html_post_processors(), ), ResponseRoute::Stream, "HTML with post-processors must route to Stream" diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 73b9419c6..22e8a1547 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -259,8 +259,9 @@ export function installGptShim(): boolean { function injectAdmIntoSlot(divId: string, adm: string): void { try { // divId may be the container div (used by GPT slot) or the inner div. - // Search both so we can find the GAM iframe wherever it was rendered. - const slotEl = document.getElementById(divId); + // Resolve it the same way the rest of adInit does (exact then prefix) so + // a config div_id prefix with a render-time suffix still finds the element. + const slotEl = findSlotElementByDivId(divId); if (!slotEl) return; // Extract the first iframe src from the adm (e.g. mocktioneer creative @@ -679,6 +680,7 @@ export function installSpaAuctionHook(): void { async function onNavigate(path: string): Promise { if (path === currentPath) return; + const previousPath = currentPath; currentPath = path; inflight?.abort(); const controller = new AbortController(); @@ -694,7 +696,14 @@ export function installSpaAuctionHook(): void { headers: { 'X-TSJS-Page-Bids': '1' }, signal: controller.signal, }); - if (!res.ok) return; + if (!res.ok) { + // A transient page-bids failure must not strand this route: roll the + // committed path back so a later navigation here retries instead of + // being skipped by the no-op guard at the top. Only roll back when no + // newer navigation has already advanced currentPath. + if (inflight === controller) currentPath = previousPath; + return; + } const data = (await res.json()) as PageBidsResponse; if (inflight !== controller) return; // Defer applying bids until the new route's ad containers exist, so a @@ -716,6 +725,7 @@ export function installSpaAuctionHook(): void { } } catch (err) { if (err instanceof DOMException && err.name === 'AbortError') return; + if (inflight === controller) currentPath = previousPath; log.warn('SPA auction hook: fetch failed', err); } } diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 40a8d9e2e..f6d55fb4a 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -270,7 +270,12 @@ function sanitizeAuctionUid(uid: { const sanitizedUid: AuctionEid['uids'][number] = { id: uid.id }; - if (Number.isInteger(uid.atype) && uid.atype >= 0 && uid.atype <= 255) { + if ( + typeof uid.atype === 'number' && + Number.isInteger(uid.atype) && + uid.atype >= 0 && + uid.atype <= 255 + ) { sanitizedUid.atype = uid.atype; } @@ -330,11 +335,18 @@ function findInjectedSlotForRefresh(slot: RefreshGptSlot): AuctionSlot | undefin return undefined; } - return window.tsjs?.adSlots?.find( - (adSlot) => - elementId === adSlot.div_id || - elementId === `${adSlot.div_id}-container` || - elementId.startsWith(adSlot.div_id) + const slots = window.tsjs?.adSlots; + if (!slots) { + return undefined; + } + + // Prefer an exact (or container) match across all slots before the prefix + // fallback, so prefix-overlapping div_ids (e.g. "ad" and "ad-header") resolve + // to the correct slot instead of the first slot whose div_id is a prefix. + return ( + slots.find( + (adSlot) => elementId === adSlot.div_id || elementId === `${adSlot.div_id}-container` + ) ?? slots.find((adSlot) => adSlot.div_id.length > 0 && elementId.startsWith(adSlot.div_id)) ); } diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 738a1cc78..6d52c368f 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -864,6 +864,63 @@ describe('prebid/installRefreshHandler', () => { ); }); + it('resolves the exact slot when div_ids share a prefix', () => { + // Regression: a single find() with a startsWith() clause returned the + // first slot whose div_id is a prefix of the element id. With div_ids + // "div-ad" and "div-ad-header", refreshing the "div-ad-header" element + // must resolve to the header slot, not the shorter prefix slot. + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'prefix_ad', + gam_unit_path: '/123/prefix', + div_id: 'div-ad', + formats: [[300, 250]], + targeting: { zone: 'prefix' }, + }, + { + id: 'header_ad', + gam_unit_path: '/123/header', + div_id: 'div-ad-header', + formats: [[970, 250]], + targeting: { zone: 'header' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-header', + mediaTypes: { + banner: { + name: 'header', + sizes: [[970, 250]], + }, + }, + }), + ], + }) + ); + }); + it('scopes the GPT targeting call to the refreshed slot code', () => { const setTargetingForGPTAsync = vi.fn(); (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; From ae9d50a2cad9eabd52e55fa390de73c48fc4a540 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 29 Jun 2026 23:03:45 +0530 Subject: [PATCH 125/395] Drop tokio from the integration-tests lockfile Moving tokio to trusted-server-core dev-dependencies removed it from the crate's normal dependency list, so the integration-tests lockfile (which resolves core's non-dev deps) no longer pins tokio under core. Keeps `cargo --locked` green for the integration job. --- crates/trusted-server-integration-tests/Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/trusted-server-integration-tests/Cargo.lock b/crates/trusted-server-integration-tests/Cargo.lock index 4f7c723d3..6fa535a57 100644 --- a/crates/trusted-server-integration-tests/Cargo.lock +++ b/crates/trusted-server-integration-tests/Cargo.lock @@ -4593,7 +4593,6 @@ dependencies = [ "serde_json", "sha2", "subtle", - "tokio", "toml", "trusted-server-js", "trusted-server-openrtb", From 802e841896c01e5e8aebf15e92d47428132cd857 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 30 Jun 2026 10:06:25 +0530 Subject: [PATCH 126/395] Run the server-side auction on the Axum, Cloudflare, and Spin adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These EdgeZero-style adapters finalize buffered, and the sync `buffer_publisher_response` drives `stream_publisher_body`, which ignores `params.dispatched_auction` — so they injected an empty `tsjs.bids = {}` while Fastly (legacy streaming finalize) served real bids. - Add `buffer_publisher_response_async` in core: for the Stream variant it drives `stream_publisher_body_async`, which awaits `collect_dispatched_auction`, writes `ad_bids_state`, and injects the bids before ``. - Pass the configured `creative_opportunities.slot` (not empty) to `handle_publisher_request` on all three adapters; it matches them against the request path internally. - Call the async finalize from each adapter (Cloudflare/Spin via their now-async `resolve_publisher_response`). EID targeting stays off for now (these adapters pass `kv: None`). --- crates/trusted-server-adapter-axum/src/app.rs | 28 +++++++-- .../src/app.rs | 46 ++++++++++++--- crates/trusted-server-adapter-spin/src/app.rs | 51 ++++++++++++---- crates/trusted-server-core/src/publisher.rs | 59 +++++++++++++++++++ 4 files changed, 158 insertions(+), 26 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 8cd53d48f..ceab9eac4 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -19,7 +19,7 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, buffer_publisher_response, handle_publisher_request, handle_tsjs_dynamic, + AuctionDispatch, buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -166,15 +166,21 @@ async fn dispatch_fallback( }); } - // Server-side auction is deferred for the EdgeZero adapters: pass no slots - // so `handle_publisher_request` dispatches no auction. + // Run the server-side auction with the configured creative-opportunity + // slots; `handle_publisher_request` matches them against the request path. let mut ec_context = EcContext::default(); + let slots = state + .settings + .creative_opportunities + .as_ref() + .map(|co| co.slot.as_slice()) + .unwrap_or(&[]); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots: &[], + slots, registry: None, }; - handle_publisher_request( + let publisher_response = handle_publisher_request( &state.settings, services, None, @@ -182,8 +188,18 @@ async fn dispatch_fallback( auction, req, ) + .await?; + // Async finalize so the dispatched auction is collected and its bids are + // injected before `` (the sync buffer path would drop them). + buffer_publisher_response_async( + publisher_response, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + services, + ) .await - .and_then(|pr| buffer_publisher_response(pr, &method, &state.settings, &state.registry)) } fn fallback_handler( diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index d4e8fb3d6..e62f3a535 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -19,7 +19,7 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PublisherResponse, buffer_publisher_response, handle_publisher_request, + AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, }; use trusted_server_core::request_signing::{ @@ -118,16 +118,27 @@ where /// Collapse a [`PublisherResponse`] into a plain [`Response`]. /// -/// Delegates to the shared [`buffer_publisher_response`], which enforces +/// Delegates to the shared [`buffer_publisher_response_async`], which collects +/// the dispatched server-side auction and enforces /// `settings.publisher.max_buffered_body_bytes`, then removes any /// `Transfer-Encoding` header since the buffered body is no longer chunked. -fn resolve_publisher_response( +async fn resolve_publisher_response( publisher_response: PublisherResponse, method: &Method, settings: &Settings, registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, ) -> Result> { - let mut response = buffer_publisher_response(publisher_response, method, settings, registry)?; + let mut response = buffer_publisher_response_async( + publisher_response, + method, + settings, + registry, + orchestrator, + services, + ) + .await?; response.headers_mut().remove(header::TRANSFER_ENCODING); Ok(response) } @@ -298,12 +309,18 @@ fn build_router(state: &Arc) -> RouterService { }) } else { let mut ec_context = EcContext::default(); + let slots = state + .settings + .creative_opportunities + .as_ref() + .map(|co| co.slot.as_slice()) + .unwrap_or(&[]); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots: &[], + slots, registry: None, }; - handle_publisher_request( + match handle_publisher_request( &state.settings, &services, None, @@ -312,9 +329,20 @@ fn build_router(state: &Arc) -> RouterService { req, ) .await - .and_then(|pr| { - resolve_publisher_response(pr, &method, &state.settings, &state.registry) - }) + { + Ok(pr) => { + resolve_publisher_response( + pr, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &services, + ) + .await + } + Err(e) => Err(e), + } }; Ok(result.unwrap_or_else(|e| http_error(&e))) diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 287bbfd8a..26852eacb 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -14,12 +14,13 @@ use trusted_server_core::ec::EcContext; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; +use trusted_server_core::platform::RuntimeServices; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PublisherResponse, buffer_publisher_response, handle_publisher_request, + AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, }; use trusted_server_core::request_signing::{ @@ -79,16 +80,27 @@ fn build_state_with_settings( /// Collapse a [`PublisherResponse`] into a plain [`Response`]. /// -/// Delegates to the shared [`buffer_publisher_response`], which enforces -/// `settings.publisher.max_buffered_body_bytes` so a large processable -/// origin response fails safely instead of exhausting the Wasm heap. -fn resolve_publisher_response( +/// Delegates to the shared [`buffer_publisher_response_async`], which collects +/// the dispatched server-side auction and enforces +/// `settings.publisher.max_buffered_body_bytes` so a large processable origin +/// response fails safely instead of exhausting the Wasm heap. +async fn resolve_publisher_response( publisher_response: PublisherResponse, method: &Method, settings: &Settings, registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, ) -> Result> { - buffer_publisher_response(publisher_response, method, settings, registry) + buffer_publisher_response_async( + publisher_response, + method, + settings, + registry, + orchestrator, + services, + ) + .await } // --------------------------------------------------------------------------- @@ -600,12 +612,18 @@ fn build_router(state: &Arc) -> RouterService { }) } else { let mut ec_context = EcContext::default(); + let slots = state + .settings + .creative_opportunities + .as_ref() + .map(|co| co.slot.as_slice()) + .unwrap_or(&[]); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots: &[], + slots, registry: None, }; - handle_publisher_request( + match handle_publisher_request( &state.settings, &services, None, @@ -614,9 +632,20 @@ fn build_router(state: &Arc) -> RouterService { req, ) .await - .and_then(|pr| { - resolve_publisher_response(pr, &method, &state.settings, &state.registry) - }) + { + Ok(pr) => { + resolve_publisher_response( + pr, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &services, + ) + .await + } + Err(e) => Err(e), + } }; Ok(result.unwrap_or_else(|e| http_error(&e))) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index b85a5ad22..4f6f7823d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -495,6 +495,65 @@ pub fn buffer_publisher_response( } } +/// Async variant of [`buffer_publisher_response`] that collects the dispatched +/// server-side auction before buffering. +/// +/// The sync [`buffer_publisher_response`] drives [`stream_publisher_body`], +/// which ignores `params.dispatched_auction`, so its `` injection always +/// falls back to empty `tsjs.bids`. Adapters that finalize on an async runtime +/// (Axum, Cloudflare, Spin) call this instead: it drives +/// [`stream_publisher_body_async`], which awaits +/// [`AuctionOrchestrator::collect_dispatched_auction`], writes the winning bids +/// into `ad_bids_state`, and injects them before ``. +/// +/// # Errors +/// +/// Returns an error if the streaming pipeline fails to process the response +/// body, or if the processed body exceeds the configured buffer cap. +pub async fn buffer_publisher_response_async( + publisher_response: PublisherResponse, + method: &Method, + settings: &Settings, + integration_registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, +) -> Result, Report> { + match publisher_response { + PublisherResponse::Buffered(response) => Ok(response), + PublisherResponse::Stream { + mut response, + body, + mut params, + } => { + if !response_carries_body(method, response.status()) { + return Ok(response); + } + let mut output = BoundedWriter::new(settings.publisher.max_buffered_body_bytes); + stream_publisher_body_async( + body, + &mut output, + &mut params, + settings, + integration_registry, + orchestrator, + services, + ) + .await?; + let bytes = output.into_inner(); + response.headers_mut().insert( + http::header::CONTENT_LENGTH, + http::HeaderValue::from(bytes.len() as u64), + ); + *response.body_mut() = EdgeBody::from(bytes); + Ok(response) + } + PublisherResponse::PassThrough { mut response, body } => { + *response.body_mut() = body; + Ok(response) + } + } +} + /// Returns `true` when a buffered publisher response should carry a body and a /// recomputed `Content-Length`. /// From 297efcd6f7e06dfaa58e9fefcb37f8f9ee710bba Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 30 Jun 2026 10:42:16 +0530 Subject: [PATCH 127/395] Build the EC consent context from the request on the portability adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auction consent gate (`consent_allows_server_side_auction`) reads jurisdiction and TCF consent from the EC context. The adapters passed `EcContext::default()` to `handle_publisher_request`, leaving jurisdiction Unknown with no consent — so the gate failed closed and no auction ran (empty `tsjs.bids`), even though the slots matched. Build the context via `read_from_request_with_geo` (consent from the request, geo from the platform), mirroring the Fastly entry point, and fall back to default on a parse error. Cloudflare resolves geo from the Workers `cf` object when deployed; Axum and Spin have no-op geo providers, so on those a known non-GDPR jurisdiction requires the request to carry geo or the gate needs a TCF consent signal. --- crates/trusted-server-adapter-axum/src/app.rs | 16 ++++++++++++++- .../src/app.rs | 19 +++++++++++++++++- crates/trusted-server-adapter-spin/src/app.rs | 20 ++++++++++++++++++- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index ceab9eac4..953a0839f 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -168,7 +168,21 @@ async fn dispatch_fallback( // Run the server-side auction with the configured creative-opportunity // slots; `handle_publisher_request` matches them against the request path. - let mut ec_context = EcContext::default(); + // Build the EC context (consent + jurisdiction) from the request like the + // Fastly entry point — `EcContext::default()` leaves jurisdiction Unknown, + // which fails the auction consent gate closed. Geo comes from the platform + // (no-op on the local Axum dev server, so jurisdiction stays Unknown there + // unless the request carries TCF consent). + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + let mut ec_context = + EcContext::read_from_request_with_geo(&state.settings, &req, services, geo_info.as_ref()) + .unwrap_or_default(); let slots = state .settings .creative_opportunities diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index e62f3a535..670c09255 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -308,7 +308,24 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - let mut ec_context = EcContext::default(); + // Build the EC context (consent + jurisdiction) from the request + // like the Fastly entry point — `EcContext::default()` leaves + // jurisdiction Unknown and fails the auction consent gate closed. + // Geo comes from the Workers `cf` object when deployed. + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + let mut ec_context = EcContext::read_from_request_with_geo( + &state.settings, + &req, + &services, + geo_info.as_ref(), + ) + .unwrap_or_default(); let slots = state .settings .creative_opportunities diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 26852eacb..d8a487d20 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -611,7 +611,25 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - let mut ec_context = EcContext::default(); + // Build the EC context (consent + jurisdiction) from the request + // like the Fastly entry point — `EcContext::default()` leaves + // jurisdiction Unknown and fails the auction consent gate closed. + // Spin's platform geo is a no-op, so jurisdiction stays Unknown + // unless the request carries TCF consent. + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + let mut ec_context = EcContext::read_from_request_with_geo( + &state.settings, + &req, + &services, + geo_info.as_ref(), + ) + .unwrap_or_default(); let slots = state .settings .creative_opportunities From c24cf5271d5236dcb8ebac81d13f80e0a3d237fd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 30 Jun 2026 10:46:13 +0530 Subject: [PATCH 128/395] Log the server-side ad-stack gate inputs at debug When the auction does not run, this pinpoints which gate suppressed it (slots, bot, navigation, consent, or orchestrator kill switch) instead of only seeing `dispatch_auction: None`. Pair with the EC-context jurisdiction log when consent_allows_auction is false. --- crates/trusted-server-core/src/publisher.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4f6f7823d..d0d7ef812 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1322,6 +1322,17 @@ pub async fn handle_publisher_request( auction.orchestrator.is_enabled(), ); let should_run_auction = should_run_ad_stack; + // Diagnostic: shows which gate suppresses the server-side auction. Pair with + // the `EC context: ... jurisdiction=...` line from EC-context construction + // when `consent_allows_auction=false`. + log::debug!( + "server-side ad-stack gate: is_get={is_get} is_navigation={is_navigation} \ + is_prefetch={is_prefetch} is_bot={is_bot} matched_slots={} \ + consent_allows_auction={consent_allows_auction} orchestrator_enabled={} \ + -> should_run_auction={should_run_auction}", + matched_slots.len(), + auction.orchestrator.is_enabled(), + ); if matched_slots.is_empty() && settings.creative_opportunities.is_some() { log::debug!( From 36a6e7ae58f05f7bd0bc4c2b4fbcefb8e3efa3ed Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 30 Jun 2026 15:37:21 +0530 Subject: [PATCH 129/395] Run the server-side auction on the Fastly EdgeZero path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EdgeZero buffered path passed empty slots and finalized via the sync `buffer_publisher_response`, so configured creative-opportunity slots were routed to the legacy path by `edgezero_can_handle_settings`. Now that `buffer_publisher_response_async` collects the dispatched auction, EdgeZero can run the full ad stack: - Pass the configured `creative_opportunities.slot` and finalize via `buffer_publisher_response_async` (the path's `ec.ec_context` already carries consent + platform geo). EID targeting stays off (`registry: None`). - Drop the `edgezero_can_handle_settings` gate, its routing branch, the three tests, and the now-unused test settings helpers — EdgeZero handles configured slots, so the legacy fallback for them is obsolete. --- .../trusted-server-adapter-fastly/src/app.rs | 47 ++++--- .../trusted-server-adapter-fastly/src/main.rs | 122 +----------------- 2 files changed, 31 insertions(+), 138 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 13a348314..465a9fa54 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -114,7 +114,7 @@ use trusted_server_core::proxy::{ AssetProxyCachePolicy, }; use trusted_server_core::publisher::{ - buffer_publisher_response, handle_publisher_request, handle_tsjs_dynamic, BoundedWriter, + buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, BoundedWriter, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -715,19 +715,24 @@ async fn dispatch_fallback( // be opened, matching legacy behavior. match runtime_services_for_consent_route(&state.settings, services) { Ok(publisher_services) => { - // Server-side auction is not yet wired into the EdgeZero buffered - // finalize path (`buffer_publisher_response` runs the - // synchronous pipeline, which does not collect dispatched SSP - // bids). Pass no slots so `handle_publisher_request` dispatches no - // auction and no bid requests are wasted. The legacy path runs the - // full server-side auction; wiring it here is deferred to the - // EdgeZero cutover. + // Run the server-side auction with the configured creative- + // opportunity slots and collect the dispatched bids in the + // buffered finalize (`buffer_publisher_response_async`), matching + // the legacy streaming path. `handle_publisher_request` matches the + // slots against the request path. EID targeting stays off here + // (`registry: None`) until per-platform KV enrichment is wired. + let slots = state + .settings + .creative_opportunities + .as_ref() + .map(|creative_opportunities| creative_opportunities.slot.as_slice()) + .unwrap_or(&[]); let auction = trusted_server_core::publisher::AuctionDispatch { orchestrator: &state.orchestrator, - slots: &[], + slots, registry: None, }; - handle_publisher_request( + match handle_publisher_request( &state.settings, &publisher_services, ec.kv_graph.as_ref(), @@ -736,14 +741,20 @@ async fn dispatch_fallback( req, ) .await - .and_then(|pub_response| { - buffer_publisher_response( - pub_response, - &method, - &state.settings, - &state.registry, - ) - }) + { + Ok(pub_response) => { + buffer_publisher_response_async( + pub_response, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &publisher_services, + ) + .await + } + Err(e) => Err(e), + } } Err(e) => Err(e), } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 30e91b01f..0d79193a0 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -191,13 +191,6 @@ fn is_edgezero_enabled(config_store: &ConfigStoreHandle) -> Result bool { - settings - .creative_opportunities - .as_ref() - .is_none_or(|creative_opportunities| creative_opportunities.slot.is_empty()) -} - /// Reads `edgezero_rollout_pct` from the config store. /// /// | Config store state | Return value | Effect | @@ -352,24 +345,8 @@ fn main() { }; if route_to_edgezero { - match get_settings() { - Ok(settings) if edgezero_can_handle_settings(&settings) => { - log::debug!("routing request through EdgeZero path"); - edgezero_main(req, edgezero_config_store); - } - Ok(_) => { - log::warn!( - "EdgeZero path does not yet support configured creative_opportunity slots; routing through legacy path" - ); - legacy_main(req); - } - Err(e) => { - log::warn!( - "failed to load settings for EdgeZero compatibility check, falling back to legacy path: {e:?}" - ); - legacy_main(req); - } - } + log::debug!("routing request through EdgeZero path"); + edgezero_main(req, edgezero_config_store); } else { legacy_main(req); } @@ -1513,71 +1490,6 @@ mod tests { .expect("should parse test settings") } - fn test_settings_with_empty_creative_opportunities() -> Settings { - Settings::from_toml( - r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - - [publisher] - domain = "test-publisher.com" - cookie_domain = ".test-publisher.com" - origin_url = "https://origin.test-publisher.com" - proxy_secret = "unit-test-proxy-secret" - - [ec] - passphrase = "test-secret-key-32-bytes-minimum" - - [request_signing] - enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" - - [creative_opportunities] - gam_network_id = "12345" - auction_timeout_ms = 500 - "#, - ) - .expect("should parse test settings with creative opportunities") - } - - fn test_settings_with_configured_creative_opportunities() -> Settings { - Settings::from_toml( - r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - - [publisher] - domain = "test-publisher.com" - cookie_domain = ".test-publisher.com" - origin_url = "https://origin.test-publisher.com" - proxy_secret = "unit-test-proxy-secret" - - [ec] - passphrase = "test-secret-key-32-bytes-minimum" - - [request_signing] - enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" - - [creative_opportunities] - gam_network_id = "12345" - auction_timeout_ms = 500 - - [[creative_opportunities.slot]] - id = "atf" - page_patterns = ["/article/*"] - formats = [{ width = 300, height = 250 }] - "#, - ) - .expect("should parse test settings with configured creative opportunities") - } - #[test] fn parses_true_flag_values() { assert!(parse_edgezero_flag("true"), "should parse 'true'"); @@ -1925,36 +1837,6 @@ mod tests { ); } - #[test] - fn edgezero_accepts_settings_without_creative_opportunities() { - let settings = test_settings(); - - assert!( - edgezero_can_handle_settings(&settings), - "should allow EdgeZero when server-side ad templates are not configured" - ); - } - - #[test] - fn edgezero_accepts_settings_with_empty_creative_opportunities() { - let settings = test_settings_with_empty_creative_opportunities(); - - assert!( - edgezero_can_handle_settings(&settings), - "should allow EdgeZero when server-side ad templates are configured but no slots are enabled" - ); - } - - #[test] - fn edgezero_rejects_settings_with_configured_creative_opportunity_slots() { - let settings = test_settings_with_configured_creative_opportunities(); - - assert!( - !edgezero_can_handle_settings(&settings), - "should route through legacy path while EdgeZero lacks server-side ad-template support" - ); - } - #[test] fn health_response_short_circuits_get_health() { let req = FastlyRequest::get("https://example.com/health"); From c65c2967fd9d018b28da75cca7f2e9279336e352 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 30 Jun 2026 15:47:39 +0530 Subject: [PATCH 130/395] Enrich Fastly EdgeZero auction bids with server-side EIDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EdgeZero publisher path dispatched the auction with registry: None, so the bid request carried no KV identity-graph EIDs (only client cookie EIDs). It already passes ec.kv_graph as the identity KV, so wire the matching PartnerRegistry::from_config(settings.ec.partners) into the AuctionDispatch to resolve server-side partner EIDs — matching the legacy auction path. Fastly-only: the sync EC identity graph (KvIdentityGraph/EcKvStore) works on Fastly's sync KV; the async-KV portability adapters are unaffected (they still pass registry: None until the EC graph supports async stores). --- .../trusted-server-adapter-fastly/src/app.rs | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 465a9fa54..8267f55b9 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -719,39 +719,45 @@ async fn dispatch_fallback( // opportunity slots and collect the dispatched bids in the // buffered finalize (`buffer_publisher_response_async`), matching // the legacy streaming path. `handle_publisher_request` matches the - // slots against the request path. EID targeting stays off here - // (`registry: None`) until per-platform KV enrichment is wired. + // slots against the request path. The partner registry plus the + // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with + // server-side EIDs, same as the legacy auction. let slots = state .settings .creative_opportunities .as_ref() .map(|creative_opportunities| creative_opportunities.slot.as_slice()) .unwrap_or(&[]); - let auction = trusted_server_core::publisher::AuctionDispatch { - orchestrator: &state.orchestrator, - slots, - registry: None, - }; - match handle_publisher_request( - &state.settings, - &publisher_services, - ec.kv_graph.as_ref(), - &mut ec.ec_context, - auction, - req, - ) - .await - { - Ok(pub_response) => { - buffer_publisher_response_async( - pub_response, - &method, + match PartnerRegistry::from_config(&state.settings.ec.partners) { + Ok(partner_registry) => { + let auction = trusted_server_core::publisher::AuctionDispatch { + orchestrator: &state.orchestrator, + slots, + registry: Some(&partner_registry), + }; + match handle_publisher_request( &state.settings, - &state.registry, - &state.orchestrator, &publisher_services, + ec.kv_graph.as_ref(), + &mut ec.ec_context, + auction, + req, ) .await + { + Ok(pub_response) => { + buffer_publisher_response_async( + pub_response, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &publisher_services, + ) + .await + } + Err(e) => Err(e), + } } Err(e) => Err(e), } From 9c0c4249133029c1ee03fbcb562ba98dbac7cdc3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 1 Jul 2026 11:16:46 +0530 Subject: [PATCH 131/395] Bring the server-side auction to parity across all adapters Resolve the fifth code-review pass. The blocking findings were all cross-adapter parity gaps in the server-side auction: - Build the geo-aware EC context in the /auction handlers on Axum, Cloudflare, and Spin. They passed EcContext::default(), leaving jurisdiction Unknown and failing the consent gate closed even for consented users. A shared per-adapter build_ec_context helper now serves /auction, page-bids, and the publisher fallback, and logs (rather than swallows) a malformed-consent read error. - Wire GET /__ts/page-bids and its OPTIONS->403 CSRF guard on the Fastly EdgeZero path and all three portability adapters, reusing core handle_page_bids and a shared page_bids_preflight_denied() helper. Previously it was Fastly-legacy-only, so SPA re-auction silently fell through to the origin on every other path. - Add trusted_server_core::response_privacy with the Set-Cookie cache-privacy downgrade and the uncacheable-operator-header guard, and call it from every adapter's apply_finalize_headers so a shared cache (Cloudflare) can no longer serve an operator/origin public Cache-Control on a cookie-bearing response. Also address the inline and non-blocking findings: warn on a dropped dispatched auction for bodiless responses, extract build_slot_json shared by the initial-page and page-bids paths, use creative_opportunity_slots() everywhere, drop the PBS id->ad_id fallback, log APS slot-id collisions, align the parallel provider parse with the collect path, remove the dead sync buffer_publisher_response, factor the mediator placeholder request, drop the unused toml dependency, guard MediaType against a future serde(default), and document the Fastly-only KV EID enrichment. JS: dedup win/billing beacons across concurrent renders, add the SSR guard to installSlimPrebidLoader, and short-circuit waitForSlotElements on an already-aborted signal. Add regression tests for the currentPath rollback and the u32::MAX format-dimension rejection. --- Cargo.lock | 1 - crates/trusted-server-adapter-axum/src/app.rs | 94 +++++--- .../src/middleware.rs | 23 +- .../src/app.rs | 81 ++++--- .../src/middleware.rs | 24 +-- .../trusted-server-adapter-fastly/Cargo.toml | 1 - .../trusted-server-adapter-fastly/src/app.rs | 52 ++++- .../src/middleware.rs | 93 ++------ crates/trusted-server-adapter-spin/src/app.rs | 97 ++++++--- .../src/middleware.rs | 23 +- .../src/auction/orchestrator.rs | 9 +- .../trusted-server-core/src/auction/types.rs | 5 + .../src/creative_slot_build_check.rs | 14 ++ .../src/integrations/aps.rs | 15 +- .../src/integrations/prebid.rs | 5 +- crates/trusted-server-core/src/lib.rs | 1 + crates/trusted-server-core/src/publisher.rs | 202 +++++++++--------- .../src/response_privacy.rs | 180 ++++++++++++++++ .../lib/src/integrations/gpt/index.ts | 19 ++ .../test/integrations/gpt/spa_hook.test.ts | 33 +++ trusted-server.toml | 5 +- 21 files changed, 654 insertions(+), 323 deletions(-) create mode 100644 crates/trusted-server-core/src/response_privacy.rs diff --git a/Cargo.lock b/Cargo.lock index e026ef3af..629e5df60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3713,7 +3713,6 @@ dependencies = [ "log-fastly", "serde", "serde_json", - "toml", "trusted-server-core", "url", "urlencoding", diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 953a0839f..18548c631 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -19,7 +19,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, + AuctionDispatch, buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -129,6 +130,34 @@ where .unwrap_or_else(|e| http_error(&e))) } +// --------------------------------------------------------------------------- +// EC context +// --------------------------------------------------------------------------- + +/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, +/// `/__ts/page-bids`, and the publisher fallback). +/// +/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction +/// Unknown, which fails the auction consent gate closed even for consented +/// users. Geo comes from the platform (a no-op on the local Axum dev server, so +/// jurisdiction stays Unknown there unless the request carries TCF consent). A +/// malformed consent string is logged and falls back to the default +/// (fail-closed) context rather than being silently swallowed. +fn build_ec_context(state: &AppState, services: &RuntimeServices, req: &Request) -> EcContext { + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + EcContext::read_from_request_with_geo(&state.settings, req, services, geo_info.as_ref()) + .unwrap_or_else(|e| { + log::warn!("EC context read failed: {e:?}"); + EcContext::default() + }) +} + // --------------------------------------------------------------------------- // Fallback dispatcher (tsjs / integration proxy / publisher) // --------------------------------------------------------------------------- @@ -168,30 +197,10 @@ async fn dispatch_fallback( // Run the server-side auction with the configured creative-opportunity // slots; `handle_publisher_request` matches them against the request path. - // Build the EC context (consent + jurisdiction) from the request like the - // Fastly entry point — `EcContext::default()` leaves jurisdiction Unknown, - // which fails the auction consent gate closed. Geo comes from the platform - // (no-op on the local Axum dev server, so jurisdiction stays Unknown there - // unless the request carries TCF consent). - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed: {e}"); - None - }); - let mut ec_context = - EcContext::read_from_request_with_geo(&state.settings, &req, services, geo_info.as_ref()) - .unwrap_or_default(); - let slots = state - .settings - .creative_opportunities - .as_ref() - .map(|co| co.slot.as_slice()) - .unwrap_or(&[]); + let mut ec_context = build_ec_context(state, services, &req); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots, + slots: state.settings.creative_opportunity_slots(), registry: None, }; let publisher_response = handle_publisher_request( @@ -242,6 +251,7 @@ enum NamedRouteHandler { /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, Auction, + PageBids, FirstPartyProxy, FirstPartyClick, FirstPartySign, @@ -264,7 +274,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_routes() -> [NamedRoute; 11] { +fn named_routes() -> [NamedRoute; 12] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -310,6 +320,13 @@ fn named_routes() -> [NamedRoute; 11] { primary_methods: &[Method::POST], handler: NamedRouteHandler::Auction, }, + // GET runs the SPA re-auction; OPTIONS is denied in-handler as a CORS + // preflight guard for this side-effecting endpoint. + NamedRoute { + path: "/__ts/page-bids", + primary_methods: &[Method::GET, Method::OPTIONS], + handler: NamedRouteHandler::PageBids, + }, NamedRoute { path: "/first-party/proxy", primary_methods: &[Method::GET], @@ -368,7 +385,10 @@ fn named_route_handler( } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::Auction => { - let ec_context = EcContext::default(); + // Build the geo-aware EC context so the auction consent + // gate sees the caller's jurisdiction — `EcContext::default()` + // fails it closed for consented users. + let ec_context = build_ec_context(&state, &services, &req); handle_auction( &state.settings, &state.orchestrator, @@ -380,6 +400,30 @@ fn named_route_handler( ) .await } + NamedRouteHandler::PageBids => { + // SPA re-auction endpoint. `OPTIONS` is a CORS preflight + // for this side-effecting GET and is always denied so the + // GET handler's `X-TSJS-Page-Bids` gate stays trustworthy. + if req.method() == Method::OPTIONS { + Ok(page_bids_preflight_denied()) + } else { + let ec_context = build_ec_context(&state, &services, &req); + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + handle_page_bids( + &state.settings, + &services, + None, + auction, + &ec_context, + req, + ) + .await + } + } NamedRouteHandler::FirstPartyProxy => { handle_first_party_proxy(&state.settings, &services, req).await } diff --git a/crates/trusted-server-adapter-axum/src/middleware.rs b/crates/trusted-server-adapter-axum/src/middleware.rs index 8ad362a97..45cbedc2c 100644 --- a/crates/trusted-server-adapter-axum/src/middleware.rs +++ b/crates/trusted-server-adapter-axum/src/middleware.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response}; +use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; @@ -88,26 +88,19 @@ impl Middleware for AuthMiddleware { /// /// Unlike the Fastly variant, geo is always unavailable so `X-Geo-Info-Available: false` /// is unconditionally emitted. Fastly-specific headers are omitted. -/// Operator-configured `settings.response_headers` are applied last and can override -/// any managed header. +/// Operator-configured `settings.response_headers` are applied last (with the +/// shared cookie cache-privacy hardening) and can override any managed header. pub(crate) fn apply_finalize_headers(settings: &Settings, response: &mut Response) { response.headers_mut().insert( HEADER_X_GEO_INFO_AVAILABLE, HeaderValue::from_static("false"), ); - for (key, value) in &settings.response_headers { - let header_name = HeaderName::from_bytes(key.as_bytes()); - let header_value = HeaderValue::from_str(value); - if let (Ok(header_name), Ok(header_value)) = (header_name, header_value) { - response.headers_mut().insert(header_name, header_value); - } else { - log::warn!( - "Skipping invalid configured response header value for {}", - key - ); - } - } + // Cookie-bearing responses stay private to shared caches and operator + // headers cannot re-enable caching for uncacheable per-user payloads. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 670c09255..1a58f4ef5 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -19,8 +19,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_publisher_request, - handle_tsjs_dynamic, + AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_page_bids, + handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -81,6 +81,29 @@ fn build_per_request_services(ctx: &RequestContext) -> RuntimeServices { build_runtime_services(ctx) } +/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, +/// `/__ts/page-bids`, and the publisher fallback). +/// +/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction +/// Unknown, which fails the auction consent gate closed even for consented +/// users. Geo comes from the Workers `cf` object when deployed. A malformed +/// consent string is logged and falls back to the default (fail-closed) context +/// rather than being silently swallowed. +fn build_ec_context(settings: &Settings, services: &RuntimeServices, req: &Request) -> EcContext { + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) + .unwrap_or_else(|e| { + log::warn!("EC context read failed: {e:?}"); + EcContext::default() + }) +} + // --------------------------------------------------------------------------- // Handler factory // --------------------------------------------------------------------------- @@ -308,33 +331,10 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - // Build the EC context (consent + jurisdiction) from the request - // like the Fastly entry point — `EcContext::default()` leaves - // jurisdiction Unknown and fails the auction consent gate closed. - // Geo comes from the Workers `cf` object when deployed. - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed: {e}"); - None - }); - let mut ec_context = EcContext::read_from_request_with_geo( - &state.settings, - &req, - &services, - geo_info.as_ref(), - ) - .unwrap_or_default(); - let slots = state - .settings - .creative_opportunities - .as_ref() - .map(|co| co.slot.as_slice()) - .unwrap_or(&[]); + let mut ec_context = build_ec_context(&state.settings, &services, &req); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots, + slots: state.settings.creative_opportunity_slots(), registry: None, }; match handle_publisher_request( @@ -413,7 +413,10 @@ fn build_router(state: &Arc) -> RouterService { .post( "/auction", make_handler(Arc::clone(&state), |s, services, req| async move { - let ec_context = EcContext::default(); + // Build the geo-aware EC context so the auction consent gate + // sees the caller's jurisdiction — `EcContext::default()` + // fails it closed for consented users. + let ec_context = build_ec_context(&s.settings, &services, &req); handle_auction( &s.settings, &s.orchestrator, @@ -426,6 +429,28 @@ fn build_router(state: &Arc) -> RouterService { .await }), ) + // SPA re-auction endpoint. The OPTIONS preflight for this + // side-effecting GET is denied so the GET handler's `X-TSJS-Page-Bids` + // gate stays trustworthy. + .route( + "/__ts/page-bids", + Method::OPTIONS, + make_handler(Arc::clone(&state), |_s, _services, _req| async move { + Ok(page_bids_preflight_denied()) + }), + ) + .get( + "/__ts/page-bids", + make_handler(Arc::clone(&state), |s, services, req| async move { + let ec_context = build_ec_context(&s.settings, &services, &req); + let auction = AuctionDispatch { + orchestrator: &s.orchestrator, + slots: s.settings.creative_opportunity_slots(), + registry: None, + }; + handle_page_bids(&s.settings, &services, None, auction, &ec_context, req).await + }), + ) .get( "/first-party/proxy", make_handler(Arc::clone(&state), |s, services, req| async move { diff --git a/crates/trusted-server-adapter-cloudflare/src/middleware.rs b/crates/trusted-server-adapter-cloudflare/src/middleware.rs index 3b60cae3f..5b605bcff 100644 --- a/crates/trusted-server-adapter-cloudflare/src/middleware.rs +++ b/crates/trusted-server-adapter-cloudflare/src/middleware.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response}; +use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; @@ -96,8 +96,8 @@ impl Middleware for AuthMiddleware { /// /// `geo_available` controls `X-Geo-Info-Available`; pass `true` when /// `cf-ipcountry` was present and non-`XX` in the incoming request. -/// Operator-configured `settings.response_headers` are applied last and can -/// override any managed header. +/// Operator-configured `settings.response_headers` are applied last (with the +/// shared cookie cache-privacy hardening) and can override any managed header. pub(crate) fn apply_finalize_headers( settings: &Settings, geo_available: bool, @@ -108,18 +108,12 @@ pub(crate) fn apply_finalize_headers( HeaderValue::from_static(if geo_available { "true" } else { "false" }), ); - for (key, value) in &settings.response_headers { - let header_name = HeaderName::from_bytes(key.as_bytes()); - let header_value = HeaderValue::from_str(value); - if let (Ok(header_name), Ok(header_value)) = (header_name, header_value) { - response.headers_mut().insert(header_name, header_value); - } else { - log::warn!( - "Skipping invalid configured response header value for {}", - key - ); - } - } + // Cloudflare is a real shared cache: cookie-bearing responses must stay + // private and operator headers must not re-enable caching for uncacheable + // per-user payloads. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 91c4a36d9..8547fd519 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -21,7 +21,6 @@ log = { workspace = true } log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -toml = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 8267f55b9..990b20257 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -114,7 +114,8 @@ use trusted_server_core::proxy::{ AssetProxyCachePolicy, }; use trusted_server_core::publisher::{ - buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, BoundedWriter, + buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, AuctionDispatch, BoundedWriter, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -584,6 +585,38 @@ async fn run_named_route( ) .await } + NamedRouteHandler::PageBids => { + // SPA re-auction endpoint. `OPTIONS` is a CORS preflight for this + // side-effecting GET and is always denied so the GET handler's + // `X-TSJS-Page-Bids` gate stays trustworthy. + if req.method() == Method::OPTIONS { + return Ok(page_bids_preflight_denied()); + } + // Like the auction, page-bids reads consent data, so the consent KV + // store must be available — fail closed with 503 when configured but + // unopenable, matching legacy. + let consent_services = runtime_services_for_consent_route(&state.settings, services)?; + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let registry_ref = if partner_registry.is_empty() { + None + } else { + Some(&partner_registry) + }; + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: registry_ref, + }; + handle_page_bids( + &state.settings, + &consent_services, + ec.kv_graph.as_ref(), + auction, + &ec.ec_context, + req, + ) + .await + } NamedRouteHandler::FirstPartyProxy => { handle_first_party_proxy(&state.settings, services, req).await } @@ -722,15 +755,10 @@ async fn dispatch_fallback( // slots against the request path. The partner registry plus the // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. - let slots = state - .settings - .creative_opportunities - .as_ref() - .map(|creative_opportunities| creative_opportunities.slot.as_slice()) - .unwrap_or(&[]); + let slots = state.settings.creative_opportunity_slots(); match PartnerRegistry::from_config(&state.settings.ec.partners) { Ok(partner_registry) => { - let auction = trusted_server_core::publisher::AuctionDispatch { + let auction = AuctionDispatch { orchestrator: &state.orchestrator, slots, registry: Some(&partner_registry), @@ -977,6 +1005,7 @@ enum NamedRouteHandler { SetTester, ClearTester, Auction, + PageBids, FirstPartyProxy, FirstPartyClick, FirstPartySign, @@ -1061,6 +1090,13 @@ const NAMED_ROUTES: &[NamedRoute] = &[ primary_methods: &[Method::POST], handler: NamedRouteHandler::Auction, }, + // GET runs the SPA re-auction; OPTIONS is denied in-handler as a CORS + // preflight guard for this side-effecting endpoint. + NamedRoute { + path: "/__ts/page-bids", + primary_methods: &[Method::GET, Method::OPTIONS], + handler: NamedRouteHandler::PageBids, + }, NamedRoute { path: "/first-party/proxy", primary_methods: &[Method::GET], diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 91b7dcdec..298d30416 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -16,7 +16,7 @@ use async_trait::async_trait; use edgezero_adapter_fastly::FastlyRequestContext; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{header, HeaderName, HeaderValue, Response, StatusCode}; +use edgezero_core::http::{HeaderValue, Response, StatusCode}; use edgezero_core::middleware::{Middleware, Next}; use edgezero_core::response::IntoResponse; use std::net::IpAddr; @@ -223,84 +223,29 @@ pub(crate) fn apply_finalize_headers( } // Any response that sets a per-user cookie (notably the EC identity cookie) - // must never be shared-cached, or a shared cache could replay one user's - // Set-Cookie to others. Skip when the response is already uncacheable so we - // don't clobber a stricter directive (e.g. `no-store`). - enforce_set_cookie_cache_privacy(response); - - // Per-user responses (assembled HTML, page-bids, cookie-bearing navigations) - // carry an uncacheable Cache-Control directive (`private` or `no-store`). - // Operator headers must not re-enable shared caching for them — neither by - // replacing Cache-Control nor by reintroducing the surrogate cache headers - // the privacy paths stripped. - let response_is_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); - - for (key, value) in &settings.response_headers { - if response_is_uncacheable - && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) - || key.eq_ignore_ascii_case("surrogate-control") - || key.eq_ignore_ascii_case("fastly-surrogate-control")) - { - continue; - } - let header_name = HeaderName::from_bytes(key.as_bytes()) - .expect("should be a valid header name: response_headers validated in prepare_runtime"); - let header_value = HeaderValue::from_str(value).expect( - "should be a valid header value: response_headers validated in prepare_runtime", - ); - response.headers_mut().insert(header_name, header_value); - } + // must never be shared-cached, and per-user responses (assembled HTML, + // page-bids, cookie-bearing navigations) must not have their uncacheable + // Cache-Control re-enabled by operator headers. This shared helper runs + // byte-identically on every adapter so the privacy guarantee can't drift. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } -/// Surrogate cache headers stripped from every cookie-bearing response. A single -/// source of truth so the legacy ([`crate::enforce_set_cookie_cache_privacy`]) -/// and `EdgeZero` copies of the privacy downgrade cannot drift apart. -pub(crate) const SURROGATE_CACHE_HEADERS: &[&str] = - &["surrogate-control", "fastly-surrogate-control"]; +/// Surrogate cache headers stripped from every cookie-bearing response. +/// +/// Re-exported from [`trusted_server_core::response_privacy`] so the legacy +/// [`crate::enforce_set_cookie_cache_privacy`] `FastlyResponse` variant and the +/// shared [`Response`] downgrade cannot drift apart. +pub(crate) use trusted_server_core::response_privacy::SURROGATE_CACHE_HEADERS; /// Forces cookie-bearing responses to stay private to shared caches. /// -/// Mirrors [`crate::enforce_set_cookie_cache_privacy`] for the [`Response`] type -/// from `edgezero_core::http`. The `EdgeZero` entry point re-applies this after +/// Re-exported from [`trusted_server_core::response_privacy`] so the `EdgeZero` +/// entry point (`main.rs`) can re-apply it after /// [`ec_finalize_response`](trusted_server_core::ec::finalize::ec_finalize_response) -/// and request-filter effects, because the EC identity `Set-Cookie` is written -/// after [`apply_finalize_headers`] runs and would otherwise reach a shared cache -/// with inherited `public`/surrogate cache headers. -/// -/// Idempotent: a response already marked `private`/`no-store` keeps its stricter -/// `Cache-Control`, but the surrogate cache headers are stripped regardless so a -/// `no-store` cookie response can never retain shared cacheability. -pub(crate) fn enforce_set_cookie_cache_privacy(response: &mut Response) { - if !response.headers().contains_key(header::SET_COOKIE) { - return; - } - // Surrogate cache headers must come off every cookie-bearing response, even - // one already carrying a stricter `no-store`/`private` directive — they are - // independent of Cache-Control and would otherwise let a shared cache store - // and replay one visitor's Set-Cookie. - for name in SURROGATE_CACHE_HEADERS { - response.headers_mut().remove(*name); - } - // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match - // against a lowercased copy — `No-Store` / `Private` must count. - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); - if !already_uncacheable { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), - ); - } -} +/// writes the EC identity `Set-Cookie`, using the single shared implementation. +pub(crate) use trusted_server_core::response_privacy::enforce_set_cookie_cache_privacy; // --------------------------------------------------------------------------- // Tests @@ -317,7 +262,7 @@ mod tests { use edgezero_core::body::Body; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; - use edgezero_core::http::{request_builder, response_builder, Method, StatusCode}; + use edgezero_core::http::{request_builder, response_builder, HeaderName, Method, StatusCode}; use edgezero_core::middleware::Next; use edgezero_core::params::PathParams; use error_stack::Report; diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index d8a487d20..55e97c723 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -20,8 +20,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_publisher_request, - handle_tsjs_dynamic, + AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_page_bids, + handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -143,7 +143,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 11] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 12] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), @@ -152,6 +152,7 @@ fn named_fallback_paths() -> [(&'static str, &'static [Method]); 11] { ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), ("/auction", &[Method::POST]), + ("/__ts/page-bids", &[Method::GET, Method::OPTIONS]), ("/first-party/proxy", &[Method::GET]), ("/first-party/click", &[Method::GET]), ("/first-party/sign", &[Method::GET, Method::POST]), @@ -322,6 +323,30 @@ fn health_response() -> Response { resp } +/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, +/// `/__ts/page-bids`, and the publisher fallback). +/// +/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction +/// Unknown, which fails the auction consent gate closed even for consented +/// users. Spin's platform geo is a no-op, so jurisdiction stays Unknown unless +/// the request carries TCF consent. A malformed consent string is logged and +/// falls back to the default (fail-closed) context rather than being silently +/// swallowed. +fn build_ec_context(settings: &Settings, services: &RuntimeServices, req: &Request) -> EcContext { + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) + .unwrap_or_else(|e| { + log::warn!("EC context read failed: {e:?}"); + EcContext::default() + }) +} + // --------------------------------------------------------------------------- // Error helper // --------------------------------------------------------------------------- @@ -506,7 +531,10 @@ fn build_router(state: &Arc) -> RouterService { // OpenRTB metadata that auction signing derives from // `RequestInfo::from_request` uses the trusted runtime authority. let req = ctx.into_request(); - let ec_context = EcContext::default(); + // Build the geo-aware EC context so the auction consent gate sees + // the caller's jurisdiction — `EcContext::default()` fails it + // closed for consented users. + let ec_context = build_ec_context(&s.settings, &services, &req); Ok(handle_auction( &s.settings, &s.orchestrator, @@ -521,6 +549,33 @@ fn build_router(state: &Arc) -> RouterService { } }; + // GET /__ts/page-bids — SPA re-auction endpoint. + let s = Arc::clone(&state); + let page_bids_handler = move |ctx: RequestContext| { + let s = Arc::clone(&s); + async move { + let services = build_runtime_services(&ctx); + let req = ctx.into_request(); + let ec_context = build_ec_context(&s.settings, &services, &req); + let auction = AuctionDispatch { + orchestrator: &s.orchestrator, + slots: s.settings.creative_opportunity_slots(), + registry: None, + }; + Ok( + handle_page_bids(&s.settings, &services, None, auction, &ec_context, req) + .await + .unwrap_or_else(|e| http_error(&e)), + ) + } + }; + + // OPTIONS /__ts/page-bids — deny the CORS preflight for this + // side-effecting GET so the `X-TSJS-Page-Bids` gate stays trustworthy. + let page_bids_options_handler = |_ctx: RequestContext| async { + Ok::(page_bids_preflight_denied()) + }; + // GET /first-party/proxy let s = Arc::clone(&state); let fp_proxy_handler = move |ctx: RequestContext| { @@ -611,34 +666,10 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - // Build the EC context (consent + jurisdiction) from the request - // like the Fastly entry point — `EcContext::default()` leaves - // jurisdiction Unknown and fails the auction consent gate closed. - // Spin's platform geo is a no-op, so jurisdiction stays Unknown - // unless the request carries TCF consent. - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed: {e}"); - None - }); - let mut ec_context = EcContext::read_from_request_with_geo( - &state.settings, - &req, - &services, - geo_info.as_ref(), - ) - .unwrap_or_default(); - let slots = state - .settings - .creative_opportunities - .as_ref() - .map(|co| co.slot.as_slice()) - .unwrap_or(&[]); + let mut ec_context = build_ec_context(&state.settings, &services, &req); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots, + slots: state.settings.creative_opportunity_slots(), registry: None, }; match handle_publisher_request( @@ -708,6 +739,12 @@ fn build_router(state: &Arc) -> RouterService { .post("/_ts/admin/keys/rotate", rotate_handler) .post("/_ts/admin/keys/deactivate", deactivate_handler) .post("/auction", auction_handler) + .get("/__ts/page-bids", page_bids_handler) + .route( + "/__ts/page-bids", + Method::OPTIONS, + page_bids_options_handler, + ) .get("/first-party/proxy", fp_proxy_handler) .get("/first-party/click", fp_click_handler) .get("/first-party/sign", fp_sign_handler) diff --git a/crates/trusted-server-adapter-spin/src/middleware.rs b/crates/trusted-server-adapter-spin/src/middleware.rs index 62f83e1ea..1bcede1fc 100644 --- a/crates/trusted-server-adapter-spin/src/middleware.rs +++ b/crates/trusted-server-adapter-spin/src/middleware.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response}; +use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; @@ -124,8 +124,8 @@ impl Middleware for NormalizeMiddleware { /// /// `geo_available` controls `X-Geo-Info-Available`. Spin passes `false` /// because it has no geo headers. Operator-configured -/// `settings.response_headers` are applied last and can override any managed -/// header. +/// `settings.response_headers` are applied last (with the shared cookie +/// cache-privacy hardening) and can override any managed header. pub(crate) fn apply_finalize_headers( settings: &Settings, geo_available: bool, @@ -136,18 +136,11 @@ pub(crate) fn apply_finalize_headers( HeaderValue::from_static(if geo_available { "true" } else { "false" }), ); - for (key, value) in &settings.response_headers { - let header_name = HeaderName::from_bytes(key.as_bytes()); - let header_value = HeaderValue::from_str(value); - if let (Ok(header_name), Ok(header_value)) = (header_name, header_value) { - response.headers_mut().insert(header_name, header_value); - } else { - log::warn!( - "Skipping invalid configured response header value for {}", - key - ); - } - } + // Cookie-bearing responses stay private to shared caches and operator + // headers cannot re-enable caching for uncacheable per-user payloads. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index c29f182e2..1898a628d 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -541,7 +541,14 @@ impl AuctionOrchestrator { { let response_time_ms = start_time.elapsed().as_millis() as u64; - match provider.parse_response(response, response_time_ms).await { + // Use the context-aware parse so a provider overriding + // `parse_response_with_context` behaves identically on the + // parallel (`/auction`, page-bids) and collect (publisher) + // paths. The default impl delegates to `parse_response`. + match provider + .parse_response_with_context(response, response_time_ms, context) + .await + { Ok(auction_response) => { log::info!( "Provider '{}' returned {} bids (status: {:?}, time: {}ms)", diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index ffe918aa4..14c7713f8 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -53,6 +53,11 @@ pub struct AdFormat { } /// Media type enumeration. +/// +/// `Default` is `Banner` for programmatic construction only. Do **not** add +/// `#[serde(default)]` to any field of this type: it would coerce an +/// unknown/missing media type to `Banner` rather than failing, silently +/// mis-typing video/native slots. Deserialization must stay strict. #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum MediaType { diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index 066cdde1a..2e763083f 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -478,6 +478,20 @@ mod tests { assert!(err.contains("positive width and height"), "got: {err}"); } + #[test] + fn rejects_format_dimension_above_u32_range() { + // Runtime dimensions are `u32`; a value above `u32::MAX` would silently + // truncate when parsed into the runtime slot, so it must fail at build. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 5_000_000_000_u64, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("width above u32::MAX must fail at build time"); + assert!(err.contains("within u32 range"), "got: {err}"); + } + #[test] fn rejects_empty_page_patterns() { let slot = json!({ diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 2ba1e5149..1d2d4ea4a 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -345,7 +345,20 @@ impl ApsAuctionProvider { .and_then(|v| v.as_str()) .unwrap_or(&slot.id) .to_string(); - slot_id_map.insert(aps_slot_id.clone(), slot.id.clone()); + // Last-write-wins: two slots configuring the same + // `[bidders.aps].slotID` would remap one slot's bids to the + // wrong creative slot. Log the collision so a misconfiguration + // is diagnosable, mirroring the build_bid_index collision log. + if let Some(previous_slot_id) = + slot_id_map.insert(aps_slot_id.clone(), slot.id.clone()) + { + log::debug!( + "APS slot ID '{aps_slot_id}' maps to multiple creative slots \ + ('{previous_slot_id}' overwritten by '{}'); bids for this APS \ + slot will resolve to the last one", + slot.id, + ); + } // Extract sizes from banner formats let sizes: Vec<[u32; 2]> = slot diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 451650611..300d5dd68 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1490,9 +1490,12 @@ impl PrebidAuctionProvider { .map(std::string::ToString::to_string) }; + // `adid` is the creative/ad identifier. The OpenRTB `id` is the bid ID, + // not an ad ID, so it is not used as a fallback: surfacing it as `ad_id` + // (which is exposed raw in the debug bid) would mislead any consumer that + // treats `ad_id` as a creative identifier. Absent `adid`, `ad_id` is None. let ad_id = bid_obj .get("adid") - .or_else(|| bid_obj.get("id")) .and_then(|v| v.as_str()) .map(String::from); diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 3bcb0b652..d4ee8515d 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -62,6 +62,7 @@ pub mod proxy; pub mod publisher; pub mod redacted; pub mod request_signing; +pub mod response_privacy; pub mod rsc_flight; pub(crate) mod s3_sigv4; pub mod settings; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d0d7ef812..21c6f560d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -440,68 +440,23 @@ pub struct OwnedProcessResponseParams { pub(crate) price_granularity: PriceGranularity, } -/// Buffer a [`PublisherResponse`] into a single [`Response`]. +/// Buffers a [`PublisherResponse`] into a single [`Response`], collecting the +/// dispatched server-side auction before buffering. /// /// Handles all three variants: returns [`PublisherResponse::Buffered`] unchanged, -/// pipes [`PublisherResponse::Stream`] through the streaming pipeline into memory, -/// and reattaches [`PublisherResponse::PassThrough`] bodies directly. +/// pipes [`PublisherResponse::Stream`] through the streaming pipeline into +/// memory, and reattaches [`PublisherResponse::PassThrough`] bodies directly. /// /// The buffered size is capped by `settings.publisher.max_buffered_body_bytes` -/// (16 MiB by default), so processable origin responses cannot grow the -/// buffer without bound and exhaust the Wasm heap. +/// (16 MiB by default), so processable origin responses cannot grow the buffer +/// without bound and exhaust the Wasm heap. /// -/// `method` is used to preserve metadata for bodiless responses: `HEAD` and -/// bodiless statuses (204, 304) carry no body but may advertise the `GET` -/// representation's length. `handle_publisher_request` already strips the origin -/// `Content-Length` for processable [`PublisherResponse::Stream`] responses, so -/// rewriting it here to the buffered byte count (`0`) would replace it with a -/// misleading length. Those responses skip the buffer, the length rewrite, and -/// the body replacement, mirroring the asset path's bodiless guard. +/// `method` preserves metadata for bodiless responses: `HEAD` and bodiless +/// statuses (204, 304) carry no body but may advertise the `GET` representation's +/// length, so they skip the buffer and length rewrite. /// -/// # Errors -/// -/// Returns an error if the streaming pipeline fails to process the response -/// body, or if the processed body exceeds the configured buffer cap. -pub fn buffer_publisher_response( - publisher_response: PublisherResponse, - method: &Method, - settings: &Settings, - integration_registry: &IntegrationRegistry, -) -> Result, Report> { - match publisher_response { - PublisherResponse::Buffered(response) => Ok(response), - PublisherResponse::Stream { - mut response, - body, - params, - } => { - if !response_carries_body(method, response.status()) { - return Ok(response); - } - let mut output = BoundedWriter::new(settings.publisher.max_buffered_body_bytes); - stream_publisher_body(body, &mut output, ¶ms, settings, integration_registry)?; - let bytes = output.into_inner(); - response.headers_mut().insert( - http::header::CONTENT_LENGTH, - http::HeaderValue::from(bytes.len() as u64), - ); - *response.body_mut() = EdgeBody::from(bytes); - Ok(response) - } - PublisherResponse::PassThrough { mut response, body } => { - *response.body_mut() = body; - Ok(response) - } - } -} - -/// Async variant of [`buffer_publisher_response`] that collects the dispatched -/// server-side auction before buffering. -/// -/// The sync [`buffer_publisher_response`] drives [`stream_publisher_body`], -/// which ignores `params.dispatched_auction`, so its `` injection always -/// falls back to empty `tsjs.bids`. Adapters that finalize on an async runtime -/// (Axum, Cloudflare, Spin) call this instead: it drives +/// Every adapter (Axum, Cloudflare, Spin, and the Fastly `EdgeZero` path) calls +/// this: it drives /// [`stream_publisher_body_async`], which awaits /// [`AuctionOrchestrator::collect_dispatched_auction`], writes the winning bids /// into `ad_bids_state`, and injects them before ``. @@ -526,6 +481,17 @@ pub async fn buffer_publisher_response_async( mut params, } => { if !response_carries_body(method, response.status()) { + if params.dispatched_auction.is_some() { + // A bodiless response (HEAD navigation, 204/304) has no + // `` to inject bids into, so the dispatched SSP + // requests are wasted — surface it for quota observability, + // matching the pass-through / buffered-unmodified arms. + log::warn!( + "Server-side auction dispatched but response is bodiless (method: {}, status: {}); in-flight SSP bid requests will not be collected", + method, + response.status(), + ); + } return Ok(response); } let mut output = BoundedWriter::new(settings.publisher.max_buffered_body_bytes); @@ -685,10 +651,7 @@ pub async fn stream_publisher_body_async( if !is_html { // Non-HTML: collect auction first, then stream. There is no // to hold, so delaying the entire body until collection is acceptable. - let placeholder = Request::builder() - .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) - .body(EdgeBody::empty()) - .unwrap_or_else(|_| Request::new(EdgeBody::empty())); + let placeholder = mediator_placeholder_request(); let result = orchestrator .collect_dispatched_auction( dispatched, @@ -736,6 +699,20 @@ pub async fn stream_publisher_body_async( .await } +/// Builds the canonical mediator placeholder [`Request`] passed to the collect +/// phase via [`make_collect_context`]. +/// +/// The URI is the compile-time constant +/// [`MEDIATOR_PLACEHOLDER_URL`](crate::auction::types::MEDIATOR_PLACEHOLDER_URL), +/// so the builder is infallible; a default-URI fallback would trip +/// [`make_collect_context`]'s `debug_assert_eq!`. +fn mediator_placeholder_request() -> Request { + Request::builder() + .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) + .body(EdgeBody::empty()) + .expect("MEDIATOR_PLACEHOLDER_URL should be a valid URI") +} + /// Build a minimal [`AuctionContext`] for the collect phase. /// /// See [`AuctionContext::request`]: the orchestrator's collect path runs @@ -1127,10 +1104,7 @@ async fn collect_stream_auction( settings: &Settings, ) { log::info!("body_close_hold_loop: collecting dispatched auction before held body tail"); - let placeholder = Request::builder() - .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) - .body(EdgeBody::empty()) - .unwrap_or_else(|_| Request::new(EdgeBody::empty())); + let placeholder = mediator_placeholder_request(); let collect_ctx = make_collect_context(settings, services, &placeholder); let result = orchestrator .collect_dispatched_auction(dispatched, services, &collect_ctx) @@ -1831,6 +1805,38 @@ pub(crate) fn build_empty_bids_script() -> String { build_bids_script(&serde_json::Map::new()) } +/// Builds the client-facing JSON wire shape for one creative-opportunity slot. +/// +/// Shared verbatim by [`build_ad_slots_script`] (initial page render) and +/// [`handle_page_bids`] (SPA navigation) so the slot wire shape has a single +/// definition and the two paths cannot silently diverge. Property names match +/// what the client-side TSJS bundle expects: `gam_unit_path`, `div_id`, +/// `formats`, and `targeting`. +fn build_slot_json( + slot: &crate::creative_opportunities::CreativeOpportunitySlot, + co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, +) -> serde_json::Value { + let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); + let div_id = slot.resolved_div_id(); + let formats: Vec = slot + .formats + .iter() + .map(|f| serde_json::json!([f.width, f.height])) + .collect(); + let targeting: serde_json::Map = slot + .targeting + .iter() + .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) + .collect(); + serde_json::json!({ + "id": slot.id, + "gam_unit_path": gam_path, + "div_id": div_id, + "formats": formats, + "targeting": targeting, + }) +} + /// Build the `tsjs.adSlots` ` + + +"#; + + #[test] + fn collects_gpt_slot_from_local_fixture() { + if !chrome_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + aps_slot_ids: Vec::new(), + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: false, + collect_ad_evidence: true, + }) + .expect("should collect fixture page"); + + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence + .gpt_slots + .iter() + .any(|slot| slot.gam_unit_path == "/123/news/atf"), + "should capture the defined GPT slot" + ); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0"), + "should capture the configured-prefix DOM id" + ); + } +} diff --git a/crates/trusted-server-cli/src/audit/collector.rs b/crates/trusted-server-cli/src/audit/collector.rs index 314ae54fc..4a774c9c7 100644 --- a/crates/trusted-server-cli/src/audit/collector.rs +++ b/crates/trusted-server-cli/src/audit/collector.rs @@ -1,41 +1,184 @@ -use serde::{Deserialize, Serialize}; -use url::Url; +//! Collector abstraction shared by the generic page audit and the ad-template +//! verifier. +//! +//! Decoupling collection behind [`AuditCollector`] lets the verifier orchestration +//! (Task 9) be tested with an in-memory fake collector, with no Chrome dependency. -use crate::error::CliResult; +use std::path::PathBuf; -pub(crate) trait AuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult; +use clap::Args; + +use crate::ad_templates::compare::BrowserAdEvidence; + +/// Operator-tunable browser options shared by `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// These are audit-tool knobs, not publisher runtime config, so they live on the +/// CLI (flags / `CHROME` env) rather than in `trusted-server.toml`. +#[derive(Debug, Clone, Args)] +pub struct BrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then + /// auto-detection on `PATH` and standard install locations. + #[arg(long)] + pub chrome: Option, + /// Quiet window in milliseconds (no new network resources) that marks the + /// page settled. + #[arg(long, default_value_t = 750)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = 10_000)] + pub settle_max_ms: u64, +} + +/// A request to collect a single page. +#[derive(Debug, Clone)] +pub struct BrowserCollectRequest { + /// The URL to navigate to. + pub url: url::Url, + /// Pre-navigation init scripts (evaluate-on-new-document). Empty for a plain + /// page audit; the ad-template verifier supplies the read-only collector here. + pub init_scripts: Vec, + /// Whether to perform the deterministic scroll pass after settle. + pub scroll: bool, + /// Whether to extract `window.__tsAdTemplateEvidence` after settle/scroll. + pub collect_ad_evidence: bool, +} + +/// The result of collecting a single page. +#[derive(Debug, Clone)] +pub struct CollectedPage { + /// The final URL after redirects. + pub final_url: url::Url, + /// The page title. + pub title: String, + /// Number of `')).toBeUndefined(); + expect(safeAdmIframeSrc('blob:https://example.com/uuid')).toBeUndefined(); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 6d52c368f..9ad7945c4 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -1559,3 +1559,47 @@ describe('prebid/client-side bidders', () => { errorSpy.mockRestore(); }); }); + +describe('prebid self-init user ID module timing', () => { + const userSyncCallCount = () => + mockSetConfig.mock.calls.filter(([arg]) => arg && typeof arg === 'object' && 'userSync' in arg) + .length; + + const setReadyState = (value: DocumentReadyState) => { + Object.defineProperty(document, 'readyState', { value, configurable: true }); + }; + + beforeEach(() => { + vi.resetModules(); + mockSetConfig.mockClear(); + }); + + afterEach(() => { + setReadyState('complete'); + }); + + it('installs user ID modules immediately when the bundle loads after window load', async () => { + // The GPT slim loader appends this bundle from a window.load handler, so + // the document is already complete — a load listener would never fire. + setReadyState('complete'); + + await import('../../../src/integrations/prebid/index'); + + expect(userSyncCallCount()).toBeGreaterThan(0); + }); + + it('defers user ID modules to window load when the document is still loading', async () => { + setReadyState('loading'); + + await import('../../../src/integrations/prebid/index'); + + expect(userSyncCallCount()).toBe(0); + + window.dispatchEvent(new Event('load')); + expect(userSyncCallCount()).toBe(1); + + // { once: true } — a second load event must not reinstall. + window.dispatchEvent(new Event('load')); + expect(userSyncCallCount()).toBe(1); + }); +}); From fe198d988b6b6382f5fb242c2815d8efae793613 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 6 Jul 2026 22:00:02 +0530 Subject: [PATCH 134/395] Add ts audit ad-templates generate with multi-page slot merge Reconstruct [creative_opportunities] slots from a live page's GPT registry and gampad/ads requests, and write them into an existing trusted-server.toml in place, preserving all other sections. Slots merge across runs: --page-pattern unions patterns into a re-seen slot, existing slots are preserved, and --replace wipes. Ephemeral div-id noise (React hashes, -container, hex UUIDs) is normalized to stable prefixes so verify matches across renders, and TOML keys/strings are escaped defensively. Add --cookie to ad-templates generate and verify so a valid bot-protection clearance cookie can carry the browser audit past an origin challenge. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/audit/ad_templates.rs | 4 + .../trusted-server-cli/src/audit/browser.rs | 21 +- .../trusted-server-cli/src/audit/collector.rs | 6 + .../src/audit/generate/analyzer.rs | 6 + .../src/audit/generate/browser_collector.rs | 65 +- .../src/audit/generate/collector.rs | 27 +- .../src/audit/generate/gpt_slots.rs | 585 ++++++++++++ .../src/audit/generate/mod.rs | 875 +++++++++++++++++- crates/trusted-server-cli/src/audit/mod.rs | 106 +++ crates/trusted-server-cli/src/audit/page.rs | 1 + 10 files changed, 1679 insertions(+), 17 deletions(-) create mode 100644 crates/trusted-server-cli/src/audit/generate/gpt_slots.rs diff --git a/crates/trusted-server-cli/src/audit/ad_templates.rs b/crates/trusted-server-cli/src/audit/ad_templates.rs index ce9855ff7..c7a9416c5 100644 --- a/crates/trusted-server-cli/src/audit/ad_templates.rs +++ b/crates/trusted-server-cli/src/audit/ad_templates.rs @@ -43,6 +43,7 @@ pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result<(), String &args.urls, args.strict, args.scroll, + &args.cookies, ); let stdout = io::stdout(); @@ -71,6 +72,7 @@ fn build_report( urls: &[url::Url], strict: bool, scroll: bool, + cookies: &[(String, String)], ) -> VerificationReport { let init_script = build_init_script(creative); @@ -84,6 +86,7 @@ fn build_report( init_scripts: init_script.clone().into_iter().collect(), scroll, collect_ad_evidence: true, + cookies: cookies.to_vec(), }; match collector.collect_page(request) { @@ -459,6 +462,7 @@ mod tests { &parsed, strict, false, + &[], ) } diff --git a/crates/trusted-server-cli/src/audit/browser.rs b/crates/trusted-server-cli/src/audit/browser.rs index 1f6eb2584..d4d1d068f 100644 --- a/crates/trusted-server-cli/src/audit/browser.rs +++ b/crates/trusted-server-cli/src/audit/browser.rs @@ -1,13 +1,16 @@ //! Chrome/Chromium-backed implementation of [`AuditCollector`] using //! `chromiumoxide` (CDP). //! -//! The collector is read-only: it installs optional pre-navigation init scripts, -//! navigates, waits for the page to settle, optionally scrolls, and reads back a -//! bounded set of evidence. It never captures page HTML, cookies, or storage. +//! The collector installs optional pre-navigation init scripts, sets any +//! operator-supplied cookies, navigates, waits for the page to settle, optionally +//! scrolls, and reads back a bounded set of evidence. It never *captures* page +//! HTML, cookies, or storage; supplied cookies are only *sent* to carry an +//! existing session past origin gates. use std::time::Duration; use chromiumoxide::browser::{Browser, BrowserConfig}; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; use chromiumoxide::page::Page; use futures::StreamExt as _; @@ -251,6 +254,17 @@ async fn collect_with_browser( .map_err(|error| format!("failed to install init script: {error}"))?; } + // Set operator-supplied cookies on the context before navigating so the + // origin sees an authenticated session on the first request. Scoping each to + // the request URL lets Chrome infer domain/path. + for (name, value) in &request.cookies { + let mut cookie = CookieParam::new(name.clone(), value.clone()); + cookie.url = Some(request.url.to_string()); + page.set_cookie(cookie) + .await + .map_err(|error| format!("failed to set cookie `{name}`: {error}"))?; + } + page.goto(request.url.as_str()) .await .map_err(|error| format!("failed to navigate to {}: {error}", request.url))?; @@ -470,6 +484,7 @@ mod tests { init_scripts: vec![script], scroll: false, collect_ad_evidence: true, + cookies: Vec::new(), }) .expect("should collect fixture page"); diff --git a/crates/trusted-server-cli/src/audit/collector.rs b/crates/trusted-server-cli/src/audit/collector.rs index 4a774c9c7..aca6814ca 100644 --- a/crates/trusted-server-cli/src/audit/collector.rs +++ b/crates/trusted-server-cli/src/audit/collector.rs @@ -42,6 +42,12 @@ pub struct BrowserCollectRequest { pub scroll: bool, /// Whether to extract `window.__tsAdTemplateEvidence` after settle/scroll. pub collect_ad_evidence: bool, + /// Operator-supplied `(name, value)` cookies set on the browser context + /// before navigation, scoped to the request URL. Used to carry an existing + /// authenticated session (e.g. a valid bot-protection clearance cookie) so + /// the origin serves the real page instead of a challenge. The collector + /// only sends these; it never reads cookies back. + pub cookies: Vec<(String, String)>, } /// The result of collecting a single page. diff --git a/crates/trusted-server-cli/src/audit/generate/analyzer.rs b/crates/trusted-server-cli/src/audit/generate/analyzer.rs index 2a13a27bc..e55952e23 100644 --- a/crates/trusted-server-cli/src/audit/generate/analyzer.rs +++ b/crates/trusted-server-cli/src/audit/generate/analyzer.rs @@ -283,6 +283,7 @@ mod tests { url: "https://cdn.example.com/dynamic.js".to_string(), resource_type: Some("Script".to_string()), }], + gpt_slots: Vec::new(), warnings: vec!["partial settle".to_string()], }; @@ -319,6 +320,7 @@ mod tests { html: "HTML Title".to_string(), script_tags: Vec::new(), network_requests: Vec::new(), + gpt_slots: Vec::new(), warnings: Vec::new(), }; @@ -336,6 +338,7 @@ mod tests { html: "HTML Title".to_string(), script_tags: Vec::new(), network_requests: Vec::new(), + gpt_slots: Vec::new(), warnings: Vec::new(), }; @@ -360,6 +363,7 @@ mod tests { url: "https://cdn.example.com/prebid.js".to_string(), resource_type: Some("script".to_string()), }], + gpt_slots: Vec::new(), warnings: Vec::new(), }; @@ -394,6 +398,7 @@ mod tests { }, ], network_requests: Vec::new(), + gpt_slots: Vec::new(), warnings: Vec::new(), }; @@ -424,6 +429,7 @@ mod tests { html: "".to_string(), script_tags: Vec::new(), network_requests: Vec::new(), + gpt_slots: Vec::new(), warnings: Vec::new(), }; diff --git a/crates/trusted-server-cli/src/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/audit/generate/browser_collector.rs index c26421b88..8ec83ba0c 100644 --- a/crates/trusted-server-cli/src/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/audit/generate/browser_collector.rs @@ -2,6 +2,7 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use chromiumoxide::browser::{Browser, BrowserConfig}; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; use chromiumoxide::ArcHttpRequest; use futures::StreamExt as _; use serde::Deserialize; @@ -12,7 +13,7 @@ use url::Url; use which::which; use crate::audit::generate::collector::{ - AuditCollector, CollectedPage, CollectedRequest, CollectedScriptTag, + AuditCollector, CollectedGptSlot, CollectedPage, CollectedRequest, CollectedScriptTag, }; use crate::error::{report_error, CliResult}; @@ -29,7 +30,11 @@ const RESOURCE_TIMING_BUFFER_WARNING: &str = pub(crate) struct BrowserAuditCollector; impl AuditCollector for BrowserAuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult { + fn collect_page( + &self, + target_url: &Url, + cookies: &[(String, String)], + ) -> CliResult { let runtime = Builder::new_current_thread() .enable_all() .build() @@ -39,11 +44,14 @@ impl AuditCollector for BrowserAuditCollector { )) })?; - runtime.block_on(collect_page_via_browser_async(target_url)) + runtime.block_on(collect_page_via_browser_async(target_url, cookies)) } } -async fn collect_page_via_browser_async(target_url: &Url) -> CliResult { +async fn collect_page_via_browser_async( + target_url: &Url, + cookies: &[(String, String)], +) -> CliResult { let chrome_executable = find_browser_executable()?; let user_data_dir = TempDir::new().map_err(|error| { report_error(format!( @@ -75,7 +83,7 @@ async fn collect_page_via_browser_async(target_url: &Url) -> CliResult CliResult CliResult { let page = browser.new_page("about:blank").await.map_err(|error| { report_error(format!("failed to create browser page for audit: {error}")) })?; + // Set operator-supplied cookies before navigating so the origin sees an + // authenticated session on the first request. Scoping each to the target URL + // lets Chrome infer domain/path. + for (name, value) in cookies { + let mut cookie = CookieParam::new(name.clone(), value.clone()); + cookie.url = Some(target_url.to_string()); + page.set_cookie(cookie) + .await + .map_err(|error| report_error(format!("failed to set cookie `{name}`: {error}")))?; + } + timeout(NAVIGATION_TIMEOUT, page.goto(target_url.as_str())) .await .map_err(|_| report_error(format!("timed out navigating to `{target_url}`")))? @@ -187,6 +207,14 @@ async fn collect_page_from_browser( warnings.push(warning.to_string()); } + // Best-effort read of the live GPT slot registry. This is the authoritative + // source for slot path/div/size, so a failure here downgrades to empty + // rather than failing the whole audit. + let gpt_slots: Vec = match page.evaluate(GPT_SLOTS_SCRIPT).await { + Ok(result) => result.into_value().unwrap_or_default(), + Err(_) => Vec::new(), + }; + Ok(CollectedPage { requested_url: target_url.to_string(), final_url, @@ -206,10 +234,37 @@ async fn collect_page_from_browser( resource_type: entry.initiator_type, }) .collect(), + gpt_slots, warnings, }) } +/// Reads the live GPT slot registry into `{gam_unit_path, div_id, sizes}` rows. +/// +/// Mirrors the ad-template verifier's `getSlots()` scrape: it defends against a +/// missing or partially-initialized `googletag`, keeps only numeric sizes, and +/// drops slots without a path or div id. +const GPT_SLOTS_SCRIPT: &str = r#"() => { + try { + if (!window.googletag || typeof googletag.pubads !== 'function') return []; + const pubads = googletag.pubads(); + if (typeof pubads.getSlots !== 'function') return []; + return pubads.getSlots().map((slot) => { + const path = typeof slot.getAdUnitPath === 'function' ? slot.getAdUnitPath() : ''; + const div = typeof slot.getSlotElementId === 'function' ? slot.getSlotElementId() : ''; + const rawSizes = typeof slot.getSizes === 'function' ? (slot.getSizes() || []) : []; + const sizes = rawSizes.map((size) => + (size && typeof size.getWidth === 'function' && typeof size.getHeight === 'function') + ? [size.getWidth(), size.getHeight()] + : null + ).filter(Boolean); + return { gam_unit_path: path, div_id: div, sizes }; + }).filter((slot) => slot.gam_unit_path && slot.div_id); + } catch (error) { + return []; + } +}"#; + async fn wait_for_page_settle(page: &chromiumoxide::Page) -> CliResult { let mut elapsed = Duration::ZERO; let mut previous_count = None; diff --git a/crates/trusted-server-cli/src/audit/generate/collector.rs b/crates/trusted-server-cli/src/audit/generate/collector.rs index 314ae54fc..2a31c763b 100644 --- a/crates/trusted-server-cli/src/audit/generate/collector.rs +++ b/crates/trusted-server-cli/src/audit/generate/collector.rs @@ -4,7 +4,15 @@ use url::Url; use crate::error::CliResult; pub(crate) trait AuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult; + /// Collects a live page. `cookies` are `(name, value)` pairs set on the + /// browser context before navigation (scoped to `target_url`) so an existing + /// session — e.g. a valid bot-protection clearance cookie — can carry the + /// audit past an origin challenge. + fn collect_page( + &self, + target_url: &Url, + cookies: &[(String, String)], + ) -> CliResult; } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] @@ -15,9 +23,26 @@ pub(crate) struct CollectedPage { pub(crate) html: String, pub(crate) script_tags: Vec, pub(crate) network_requests: Vec, + /// Slots read from the live GPT registry (`googletag.pubads().getSlots()`). + /// + /// Populated at `defineSlot` time, so this captures configured slots even + /// when the ad request never fires (consent-gated or iframe-issued). + #[serde(default)] + pub(crate) gpt_slots: Vec, pub(crate) warnings: Vec, } +/// A single slot read from the page's live GPT registry. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedGptSlot { + /// The GAM ad-unit path (`slot.getAdUnitPath()`). + pub(crate) gam_unit_path: String, + /// The slot's div element id (`slot.getSlotElementId()`). + pub(crate) div_id: String, + /// Numeric `[width, height]` sizes (`slot.getSizes()`, fluid entries dropped). + pub(crate) sizes: Vec<(u32, u32)>, +} + #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] pub(crate) struct CollectedScriptTag { pub(crate) src: Option, diff --git a/crates/trusted-server-cli/src/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/audit/generate/gpt_slots.rs new file mode 100644 index 000000000..8ee09f71b --- /dev/null +++ b/crates/trusted-server-cli/src/audit/generate/gpt_slots.rs @@ -0,0 +1,585 @@ +//! Reconstructs `[creative_opportunities]` slots from a live page's GPT state. +//! +//! Two complementary sources feed the reconstruction: +//! +//! 1. The **live GPT registry** (`googletag.pubads().getSlots()`) is the primary +//! source. It exposes each defined slot's ad-unit path, div id, and sizes +//! directly, and is populated at `defineSlot` time — so it captures slots even +//! when the ad request never fires (consent-gated stacks, iframe-issued +//! requests). It carries no per-slot header-bidding signal, so Prebid is +//! inferred from page-level detection. +//! 2. Captured **`gampad/ads` requests** are a fallback for any div the registry +//! did not report. Each request URL encodes the ad-unit path (`iu_parts`), div +//! id (`dids`), sizes (`prev_iu_szs`), and targeting (`prev_scp`, which does +//! carry a per-slot Prebid signal). +//! +//! Neither source executes the page's ad-stack logic ourselves; both read state +//! the page's own GPT/Prebid setup produced. + +use std::collections::BTreeSet; +use std::sync::LazyLock; + +use regex::Regex; +use url::Url; + +use crate::audit::generate::collector::{CollectedGptSlot, CollectedRequest}; + +/// A hyphen-delimited hex hash *segment* (16+ hex chars bounded by `-` or end), +/// e.g. the UUID GPT embeds in `ad-in_content--in_content-0`. Marks the +/// start of ephemeral div-id noise, like the React `_R_` hash. The trailing +/// boundary avoids truncating a legit token that merely starts with hex-like +/// characters (only `start()` of the match is used). +static HEX_HASH_SEGMENT: LazyLock = + LazyLock::new(|| Regex::new(r"-[0-9a-f]{16,}(?:-|$)").expect("should compile hex hash regex")); + +/// Hosts that serve GPT `gampad/ads` requests. +const GAMPAD_HOSTS: &[&str] = &["securepubads.g.doubleclick.net", "pubads.g.doubleclick.net"]; + +/// Common GPT div-id prefix stripped when deriving a slot id. +const GPT_DIV_PREFIX: &str = "div-gpt-ad-"; + +/// Minimum width/height for a format to be treated as a real creative size. +/// +/// GPT encodes fluid/native aspect-ratio markers (e.g. `4x1`, `8x1`) alongside +/// pixel sizes in `prev_iu_szs`; those are not banner dimensions, so they are +/// dropped from the drafted `formats`. +const MIN_FORMAT_DIMENSION: u32 = 50; + +/// A slot reconstructed from a single GPT ad request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DiscoveredSlot { + /// Slot id derived from the div id (GPT prefix stripped). + pub(crate) id: String, + /// The HTML div id that holds the creative. + pub(crate) div_id: String, + /// The full GAM ad-unit path (e.g. `/123/desktop/homepage/leaderboard`). + pub(crate) gam_unit_path: String, + /// Candidate creative sizes as `(width, height)` pixel pairs. + pub(crate) formats: Vec<(u32, u32)>, + /// Whether the slot's targeting shows Prebid/header-bidding signals. + pub(crate) has_prebid: bool, +} + +/// The result of scanning captured requests for GPT slots. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct DiscoveredSlots { + /// GAM network id shared by the discovered slots, if any were found. + pub(crate) gam_network_id: Option, + /// The reconstructed slots, deduplicated by div id in first-seen order. + pub(crate) slots: Vec, +} + +/// Reconstructs GPT slots from the page's live registry and ad requests. +/// +/// The live registry (`googletag.pubads().getSlots()`) is the primary source: it +/// carries the authoritative path/div/size for every defined slot and is present +/// even when the ad request never fires. Captured `gampad/ads` requests are a +/// fallback for any div the registry did not report, and also supply per-slot +/// Prebid signals. Slots are deduplicated by div id in first-seen order. +/// +/// `page_has_prebid` marks registry slots as Prebid-enabled when the page as a +/// whole was detected running Prebid (the registry alone carries no such signal). +pub(crate) fn discover_gpt_slots( + registry: &[CollectedGptSlot], + requests: &[CollectedRequest], + page_has_prebid: bool, +) -> DiscoveredSlots { + let mut slots = Vec::new(); + let mut gam_network_id = None; + let mut seen_divs = BTreeSet::new(); + + for entry in registry { + let Some(slot) = slot_from_registry(entry, page_has_prebid) else { + continue; + }; + if !seen_divs.insert(slot.div_id.clone()) { + continue; + } + if gam_network_id.is_none() { + gam_network_id = network_id_from_unit_path(&slot.gam_unit_path); + } + slots.push(slot); + } + + for request in requests { + let Some((network_id, slot)) = parse_gampad_request(&request.url) else { + continue; + }; + if !seen_divs.insert(slot.div_id.clone()) { + continue; + } + if gam_network_id.is_none() { + gam_network_id = Some(network_id); + } + slots.push(slot); + } + + DiscoveredSlots { + gam_network_id, + slots, + } +} + +/// Converts a live-registry slot into a [`DiscoveredSlot`]. +/// +/// Returns `None` when the slot has no usable pixel size or its div id is a +/// multi-slot (SRA) concatenation rather than a single element. +fn slot_from_registry(entry: &CollectedGptSlot, page_has_prebid: bool) -> Option { + if is_multi_slot_div(&entry.div_id) { + return None; + } + let formats: Vec<(u32, u32)> = entry + .sizes + .iter() + .copied() + .filter(|(width, height)| *width >= MIN_FORMAT_DIMENSION && *height >= MIN_FORMAT_DIMENSION) + .collect(); + if formats.is_empty() { + return None; + } + let div_stem = normalize_div_stem(&entry.div_id); + Some(DiscoveredSlot { + id: slot_id_from_div(&div_stem), + div_id: div_stem, + gam_unit_path: entry.gam_unit_path.clone(), + formats, + has_prebid: page_has_prebid, + }) +} + +/// Whether a div id is a GPT single-request (SRA) concatenation of multiple +/// slots (joined with `~`) rather than one element. +fn is_multi_slot_div(div_id: &str) -> bool { + div_id.contains('~') +} + +/// Strips ephemeral GPT div-id noise so the stored id is stable across renders. +/// +/// Removes a trailing `-container` wrapper, then truncates at the first ephemeral +/// marker — a React SSR hash (`_R_`) or a hex-UUID segment — since both +/// change on every page load. Truncating (rather than excising) keeps the result +/// a valid **prefix** of the live div id, which is how verify matches slots. +/// +/// `div-gpt-ad-leaderboard-1` (stable) is unchanged; `ad-header-0-_R_9sl…-container` +/// → `ad-header-0`; `ad-in_content-de66…f272-in_content-0` → `ad-in_content`. +fn normalize_div_stem(div_id: &str) -> String { + let stem = div_id.strip_suffix("-container").unwrap_or(div_id); + let mut cut = stem.len(); + if let Some(pos) = stem.find("_R_") { + cut = cut.min(pos); + } + if let Some(matched) = HEX_HASH_SEGMENT.find(stem) { + cut = cut.min(matched.start()); + } + stem[..cut].trim_end_matches('-').to_string() +} + +/// Extracts the leading network id from a GAM ad-unit path (`//...`). +fn network_id_from_unit_path(path: &str) -> Option { + let segment = path.trim_start_matches('/').split('/').next()?; + (!segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| segment.to_string()) +} + +/// Parses a single `gampad/ads` request URL into `(network_id, slot)`. +/// +/// Returns `None` when the URL is not a GPT ad request or is missing the fields +/// needed to describe a slot (ad-unit path, div id, and at least one size). +fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot)> { + let url = Url::parse(raw_url).ok()?; + let host = url.host_str()?; + if !GAMPAD_HOSTS.contains(&host) || !url.path().ends_with("/gampad/ads") { + return None; + } + + let mut iu_parts = None; + let mut dids = None; + let mut sizes_raw = None; + let mut fallback_sizes_raw = None; + let mut scp = None; + for (key, value) in url.query_pairs() { + match key.as_ref() { + "iu_parts" => iu_parts = Some(value.into_owned()), + "dids" => dids = Some(value.into_owned()), + "prev_iu_szs" => sizes_raw = Some(value.into_owned()), + "pb_szs" => fallback_sizes_raw = Some(value.into_owned()), + "prev_scp" => scp = Some(value.into_owned()), + _ => {} + } + } + + let iu_parts = iu_parts?; + let mut parts = iu_parts.split(',').filter(|part| !part.is_empty()); + let network_id = parts.next()?.to_string(); + let gam_unit_path = format!("/{}", iu_parts.replace(',', "/")); + // A usable unit path needs the network id plus at least one path segment. + parts.next()?; + + let raw_div = dids? + .split(',') + .map(str::trim) + .find(|did| !did.is_empty())? + .to_string(); + if is_multi_slot_div(&raw_div) { + return None; + } + let div_id = normalize_div_stem(&raw_div); + + let formats = parse_sizes(sizes_raw.as_deref().or(fallback_sizes_raw.as_deref())?); + if formats.is_empty() { + return None; + } + + let id = slot_id_from_div(&div_id); + let has_prebid = scp.as_deref().is_some_and(scp_shows_prebid); + + Some(( + network_id, + DiscoveredSlot { + id, + div_id, + gam_unit_path, + formats, + has_prebid, + }, + )) +} + +/// Parses a GPT size list (e.g. `970x250|4x1|620x366`) into pixel pairs. +/// +/// Accepts `|` or `,` separators, ignores non-`WxH` tokens, and drops +/// fluid/native ratio markers below [`MIN_FORMAT_DIMENSION`]. +fn parse_sizes(raw: &str) -> Vec<(u32, u32)> { + let mut sizes = Vec::new(); + for token in raw.split(['|', ',']) { + let Some((width, height)) = token.trim().split_once('x') else { + continue; + }; + let (Ok(width), Ok(height)) = (width.parse::(), height.parse::()) else { + continue; + }; + if width < MIN_FORMAT_DIMENSION || height < MIN_FORMAT_DIMENSION { + continue; + } + if !sizes.contains(&(width, height)) { + sizes.push((width, height)); + } + } + sizes +} + +/// Derives a slot id from a div id by stripping the common GPT prefix. +fn slot_id_from_div(div_id: &str) -> String { + div_id + .strip_prefix(GPT_DIV_PREFIX) + .unwrap_or(div_id) + .to_string() +} + +/// Detects Prebid/header-bidding signals in a slot's `prev_scp` targeting. +fn scp_shows_prebid(scp: &str) -> bool { + let scp = scp.to_ascii_lowercase(); + scp.contains("test=prebid") || scp.contains("tude=true") || scp.contains("prebid") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A sample GPT leaderboard ad request (truncated to the fields the + /// parser reads; values are otherwise unmodified live output). + const SAMPLE_LEADERBOARD: &str = "https://securepubads.g.doubleclick.net/gampad/ads?\ + gdfp_req=1&iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C8x1%7C620x366%7C325x508%7C325x204\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=ad-loc%3Dleaderboard-1%26baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid%26tude%3Dtrue\ + &pb_szs=970x250%7C620x366"; + + fn request(url: &str) -> CollectedRequest { + CollectedRequest { + url: url.to_string(), + resource_type: Some("fetch".to_string()), + } + } + + /// Discovers slots from ad requests only (no live registry). + fn from_requests(requests: &[CollectedRequest]) -> DiscoveredSlots { + discover_gpt_slots(&[], requests, false) + } + + #[test] + fn parses_leaderboard_slot() { + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD)]); + + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_eq!(discovered.slots.len(), 1, "should find one slot"); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1", "should strip the GPT div prefix"); + assert_eq!(slot.div_id, "div-gpt-ad-leaderboard-1"); + assert_eq!( + slot.gam_unit_path, + "/123456789/desktop/homepage/leaderboard1" + ); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366), (325, 508), (325, 204)], + "should keep pixel sizes and drop 4x1/8x1 fluid markers" + ); + assert!(slot.has_prebid, "prev_scp test=prebid should flag prebid"); + } + + #[test] + fn deduplicates_refreshed_slot_requests() { + // GPT refreshes the same slot; a second identical request must not + // produce a duplicate slot. + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD), request(SAMPLE_LEADERBOARD)]); + + assert_eq!( + discovered.slots.len(), + 1, + "repeat requests for the same div should collapse" + ); + } + + #[test] + fn ignores_non_gampad_requests() { + let discovered = from_requests(&[ + request("https://securepubads.g.doubleclick.net/tag/js/gpt.js"), + request("https://cdn.example.com/app.js"), + request("https://analytics.example.com/collect?iu_parts=1%2Cfoo&dids=x"), + ]); + + assert!( + discovered.slots.is_empty(), + "only doubleclick gampad/ads requests should yield slots" + ); + assert_eq!(discovered.gam_network_id, None); + } + + #[test] + fn skips_requests_missing_sizes() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x", + )]); + + assert!( + discovered.slots.is_empty(), + "a slot with no usable size should be skipped" + ); + } + + #[test] + fn skips_requests_with_only_network_id() { + // iu_parts with just the network id yields no unit path segment. + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a bare network id is not a usable ad-unit path" + ); + } + + #[test] + fn falls_back_to_pb_szs_when_prev_iu_szs_absent() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x&pb_szs=300x250%7C728x90", + )]); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!(discovered.slots[0].formats, vec![(300, 250), (728, 90)]); + } + + fn registry_slot(path: &str, div: &str, sizes: &[(u32, u32)]) -> CollectedGptSlot { + CollectedGptSlot { + gam_unit_path: path.to_string(), + div_id: div.to_string(), + sizes: sizes.to_vec(), + } + } + + #[test] + fn reads_slots_from_live_registry() { + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250), (1, 1), (620, 366)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], true); + + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "network id should come from the unit path" + ); + assert_eq!(discovered.slots.len(), 1); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1"); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366)], + "should drop the 1x1 out-of-page marker" + ); + assert!( + slot.has_prebid, + "page-level prebid should mark registry slots" + ); + } + + #[test] + fn registry_wins_and_requests_fill_gaps() { + // The registry reports the leaderboard; a gampad request reports a + // different div that the registry missed. Both should appear once. + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250)], + )]; + let requests = vec![ + // Same div as the registry — must not duplicate. + request(SAMPLE_LEADERBOARD), + // A div the registry did not report — must be added. + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Cdesktop%2Chomepage%2Csidebar1&dids=div-gpt-ad-sidebar-1&prev_iu_szs=300x600", + ), + ]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + let ids: Vec<&str> = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect(); + assert_eq!( + ids, + vec!["leaderboard-1", "sidebar-1"], + "registry slot kept, request fills the missing div, no duplicate" + ); + } + + #[test] + fn registry_slot_without_pixel_sizes_is_skipped() { + let registry = vec![registry_slot("/123/fluid", "div-gpt-ad-fluid", &[(1, 1)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a registry slot with only fluid markers is not usable" + ); + } + + #[test] + fn normalizes_ephemeral_hash_and_container_and_dedups() { + // A framework-hashed div: the same placement appears as a hashed inner div, + // a `-container` wrapper, and re-rendered with a different hash. All must + // collapse to one stable stem. + let registry = vec![ + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_", + &[(728, 90)], + ), + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_-container", + &[(728, 90)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "hash + container variants collapse" + ); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "ephemeral React hash and -container are stripped to a stable stem" + ); + assert_eq!(discovered.slots[0].id, "ad-header-0"); + } + + #[test] + fn drops_sra_multi_slot_concatenations() { + let registry = vec![registry_slot( + "/987654321/homepage/header-0/fixed_bottom-0", + "ad-header-0-_R_9slin~ad-fixed_bottom-0-_R_ainp", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "tilde-joined SRA multi-slot divs are not real single elements" + ); + } + + #[test] + fn leaves_clean_div_ids_unchanged() { + assert_eq!( + normalize_div_stem("div-gpt-ad-leaderboard-1"), + "div-gpt-ad-leaderboard-1" + ); + } + + #[test] + fn normalizes_react_and_hex_hashes_to_stable_prefixes() { + assert_eq!( + normalize_div_stem("ad-header-0-_R_9slinpflik6lb_-container"), + "ad-header-0" + ); + let stem = + normalize_div_stem("ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0"); + assert_eq!(stem, "ad-in_content"); + assert!( + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0".starts_with(&stem), + "stem must prefix-match any re-rendered hex variant" + ); + } + + #[test] + fn hex_hash_truncation_requires_a_segment_boundary() { + // Hex UUID bounded by `-` → truncated to the stem. + assert_eq!( + normalize_div_stem("ad-x-de669245b2ea4b05826dc96f07a36272-y"), + "ad-x" + ); + // A token that merely starts with 16 hex chars (no boundary) is left intact. + assert_eq!( + normalize_div_stem("ad-de669245b2ea4b05z"), + "ad-de669245b2ea4b05z" + ); + } + + #[test] + fn hex_normalized_in_content_slots_dedup() { + // Same in_content placement, different per-render hex — one stable slot. + let registry = vec![ + registry_slot( + "/987654321/site/homepage", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + registry_slot( + "/987654321/site/homepage", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-0", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "hex variants collapse to one slot" + ); + assert_eq!(discovered.slots[0].div_id, "ad-in_content"); + } +} diff --git a/crates/trusted-server-cli/src/audit/generate/mod.rs b/crates/trusted-server-cli/src/audit/generate/mod.rs index 6d06e8698..74453f6ba 100644 --- a/crates/trusted-server-cli/src/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/audit/generate/mod.rs @@ -1,13 +1,18 @@ mod analyzer; pub(crate) mod browser_collector; pub(crate) mod collector; +mod gpt_slots; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; use serde::Serialize; +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, CreativeOpportunitySlot, +}; use url::Url; use crate::audit::generate::collector::AuditCollector; @@ -37,6 +42,11 @@ pub(crate) struct GenerateArgs { /// Overwrite existing output files. #[arg(long)] pub(crate) force: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = crate::audit::parse_cookie)] + pub(crate) cookies: Vec<(String, String)>, } const DEFAULT_JS_ASSETS_PATH: &str = "js-assets.toml"; @@ -82,6 +92,7 @@ pub(crate) struct AuditOutputs { pub(crate) artifact: AuditArtifact, pub(crate) js_assets_toml: String, pub(crate) draft_config_toml: String, + pub(crate) ad_slot_count: usize, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -97,7 +108,7 @@ pub(crate) fn run_generate( ) -> CliResult<()> { let target_url = parse_audit_url(&args.url)?; let plan = resolve_output_plan(args)?; - let collected = collector.collect_page(&target_url)?; + let collected = collector.collect_page(&target_url, &args.cookies)?; let outputs = build_audit_outputs(&collected)?; let wrote_config = plan.config_path.is_some(); let written = write_audit_outputs(&outputs, &plan)?; @@ -175,12 +186,23 @@ fn build_audit_outputs(collected: &collector::CollectedPage) -> CliResult"), outputs.artifact.js_asset_count, outputs.artifact.third_party_asset_count, + outputs.ad_slot_count, if integrations.is_empty() { "none".to_string() } else { @@ -269,7 +292,11 @@ fn write_success_summary( .map_err(|error| report_error(format!("failed to write command output: {error}"))) } -fn build_draft_config(target_url: &Url, artifact: &AuditArtifact) -> CliResult { +fn build_draft_config( + target_url: &Url, + artifact: &AuditArtifact, + slots: &gpt_slots::DiscoveredSlots, +) -> CliResult { let host = target_url .host_str() .ok_or_else(|| report_error("audited URL is missing a host"))?; @@ -353,9 +380,526 @@ fn build_draft_config(target_url: &Url, artifact: &AuditArtifact) -> CliResult String { + let path = target_url.path(); + let page_pattern = if path.is_empty() { "/" } else { path }; + + let mut out = String::from( + "\n# Slots discovered from live GPT ad requests during the audit.\n\ + # Review page_patterns and formats before validating/pushing.\n", + ); + for slot in &slots.slots { + let formats = slot + .formats + .iter() + .map(|(width, height)| format!("{{ width = {width}, height = {height} }}")) + .collect::>() + .join(", "); + out.push_str(&format!( + "\n[[creative_opportunities.slot]]\n\ + id = \"{id}\"\n\ + div_id = \"{div_id}\"\n\ + gam_unit_path = \"{gam_unit_path}\"\n\ + page_patterns = [\"{page_pattern}\"]\n\ + formats = [{formats}]\n", + id = slot.id, + div_id = slot.div_id, + gam_unit_path = slot.gam_unit_path, + )); + if slot.has_prebid { + out.push_str("[creative_opportunities.slot.providers.prebid]\nbidders = {}\n"); + } + } + out +} + +/// Runs `ts audit ad-templates generate`: scrape the live page's GPT slots and +/// rewrite only the `[creative_opportunities]` slot array in `config_path` in +/// place, preserving every other section and comment. +/// +/// # Errors +/// +/// Returns an error when the config cannot be read, the page cannot be +/// collected, no slots are discovered, or the config has no +/// `[creative_opportunities]` section to update. +#[allow(clippy::too_many_arguments, reason = "cohesive one-shot command entry")] +pub(crate) fn run_update_slots( + url: &str, + config_path: &Path, + existing_creative: Option<&CreativeOpportunitiesConfig>, + page_patterns: &[String], + replace: bool, + cookies: &[(String, String)], + dry_run: bool, + collector: &dyn AuditCollector, + out: &mut dyn Write, +) -> CliResult<()> { + let target_url = parse_audit_url(url)?; + let existing = fs::read_to_string(config_path).map_err(|error| { + report_error(format!( + "failed to read config {}: {error}", + config_path.display() + )) + })?; + + let collected = collector.collect_page(&target_url, cookies)?; + let artifact = analyze_collected_page(&collected)?; + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let discovered = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + if discovered.slots.is_empty() { + return cli_error("no ad-template slots were discovered on the page"); + } + + // Patterns for slots seen on this run: the `--page-pattern` values, or the + // audited path when none are given (preserving single-page behavior). + let run_patterns: Vec = if page_patterns.is_empty() { + vec![default_page_pattern(&target_url)] + } else { + page_patterns.to_vec() + }; + + let merged = merge_slots(existing_creative, &discovered, &run_patterns, replace); + let network_id = resolve_network_id( + existing_creative, + discovered.gam_network_id.as_deref(), + replace, + ); + let rendered_slots = render_slots(&merged); + let updated = splice_creative_slots(&existing, network_id.as_deref(), &rendered_slots)?; + + if dry_run { + writeln!(out, "{updated}") + .map_err(|error| report_error(format!("failed to write preview: {error}")))?; + return Ok(()); + } + fs::write(config_path, &updated).map_err(|error| { + report_error(format!( + "failed to write config {}: {error}", + config_path.display() + )) + })?; + writeln!( + out, + "Wrote {} slot(s) to {} ({} discovered this run)", + merged.len(), + config_path.display(), + discovered.slots.len(), + ) + .map_err(|error| report_error(format!("failed to write command output: {error}"))) +} + +/// Chooses the `gam_network_id` to write. +/// +/// The existing id is kept only when a real merge preserves existing slots. +/// On `--replace`, or when the config had no slots (e.g. a placeholder +/// `[creative_opportunities]` section), the discovered id wins — mirroring +/// [`merge_slots`], which returns discovered-only in those cases. +fn resolve_network_id( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_network_id: Option<&str>, + replace: bool, +) -> Option { + let existing_network_id = existing.map(|config| config.gam_network_id.clone()); + let preserving_existing = !replace && existing.is_some_and(|config| !config.slot.is_empty()); + if preserving_existing { + existing_network_id.or_else(|| discovered_network_id.map(str::to_string)) + } else { + discovered_network_id + .map(str::to_string) + .or(existing_network_id) + } +} + +/// The default page pattern for a scraped URL: its path, or `/` for the root. +fn default_page_pattern(target_url: &Url) -> String { + let path = target_url.path(); + if path.is_empty() { + "/".to_string() + } else { + path.to_string() + } +} + +/// A slot ready to render — the union of discovered and existing fields, without +/// the core type's `pub(crate)` compiled-pattern cache. +#[derive(Debug, Clone)] +struct RenderSlot { + id: String, + div_id: Option, + gam_unit_path: Option, + page_patterns: Vec, + /// `(width, height, non-banner media type)`. + formats: Vec<(u32, u32, Option<&'static str>)>, + floor_price: Option, + targeting: BTreeMap, + aps_slot_id: Option, + /// `Some` when the slot runs Prebid; the map is per-bidder params (often empty). + prebid_bidders: Option>, +} + +impl RenderSlot { + /// The stable identity used to match slots across runs: the div id (or slot + /// id), with any trailing `-` trimmed so hand-authored stems still match. + fn key(&self) -> String { + self.div_id + .as_deref() + .unwrap_or(&self.id) + .trim_end_matches('-') + .to_string() + } + + fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { + Self { + id: slot.id.clone(), + div_id: Some(slot.div_id.clone()), + gam_unit_path: Some(slot.gam_unit_path.clone()), + page_patterns: patterns.to_vec(), + formats: slot + .formats + .iter() + .map(|&(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: slot.has_prebid.then(BTreeMap::new), + } + } + + fn from_existing(slot: &CreativeOpportunitySlot) -> Self { + Self { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + gam_unit_path: slot.gam_unit_path.clone(), + page_patterns: slot.page_patterns.clone(), + formats: slot + .formats + .iter() + .map(|format| { + ( + format.width, + format.height, + media_type_label(&format.media_type), + ) + }) + .collect(), + floor_price: slot.floor_price, + targeting: slot + .targeting + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), + prebid_bidders: slot.providers.prebid.as_ref().map(|prebid| { + prebid + .bidders + .iter() + .map(|(name, params)| (name.clone(), params.clone())) + .collect() + }), + } + } +} + +/// The non-default (non-banner) media-type label to emit, or `None` for banner. +fn media_type_label(media_type: &MediaType) -> Option<&'static str> { + match media_type { + MediaType::Banner => None, + MediaType::Video => Some("video"), + MediaType::Native => Some("native"), + } +} + +/// Merges discovered slots into the existing slot set, keyed by [`RenderSlot::key`]. +/// +/// - `--replace` (or no existing slots): the result is exactly the discovered set. +/// - Otherwise existing slots are preserved (covering other pages / hand-tuned +/// fields); a slot re-seen this run has `run_patterns` unioned into its +/// `page_patterns`; slots seen only this run are appended. +fn merge_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered: &gpt_slots::DiscoveredSlots, + run_patterns: &[String], + replace: bool, +) -> Vec { + let discovered_slots: Vec = discovered + .slots + .iter() + .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) + .collect(); + + let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); + if replace || existing_slots.is_empty() { + return discovered_slots; + } + + let mut merged: Vec = existing_slots + .iter() + .map(RenderSlot::from_existing) + .collect(); + for slot in discovered_slots { + let key = slot.key(); + if let Some(present) = merged.iter_mut().find(|existing| existing.key() == key) { + for pattern in &slot.page_patterns { + if !present.page_patterns.contains(pattern) { + present.page_patterns.push(pattern.clone()); + } + } + } else { + merged.push(slot); + } + } + merged +} + +/// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. +fn render_slots(slots: &[RenderSlot]) -> String { + let mut out = String::from( + "\n# Slots managed by `ts audit ad-templates generate`.\n\ + # Review page_patterns and formats before validating/pushing.\n", + ); + for slot in slots { + out.push_str("\n[[creative_opportunities.slot]]\n"); + out.push_str(&format!("id = {}\n", toml_string(&slot.id))); + if let Some(div_id) = &slot.div_id { + out.push_str(&format!("div_id = {}\n", toml_string(div_id))); + } + if let Some(path) = &slot.gam_unit_path { + out.push_str(&format!("gam_unit_path = {}\n", toml_string(path))); + } + let patterns = slot + .page_patterns + .iter() + .map(|pattern| toml_string(pattern)) + .collect::>() + .join(", "); + out.push_str(&format!("page_patterns = [{patterns}]\n")); + let formats = slot + .formats + .iter() + .map(|(width, height, media_type)| match media_type { + Some(kind) => { + format!("{{ width = {width}, height = {height}, media_type = \"{kind}\" }}") + } + None => format!("{{ width = {width}, height = {height} }}"), + }) + .collect::>() + .join(", "); + out.push_str(&format!("formats = [{formats}]\n")); + if let Some(floor) = slot.floor_price { + out.push_str(&format!("floor_price = {floor}\n")); + } + if !slot.targeting.is_empty() { + let pairs = slot + .targeting + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_string(value))) + .collect::>() + .join(", "); + out.push_str(&format!("targeting = {{ {pairs} }}\n")); + } + if let Some(slot_id) = &slot.aps_slot_id { + out.push_str("[creative_opportunities.slot.providers.aps]\n"); + out.push_str(&format!("slot_id = {}\n", toml_string(slot_id))); + } + if let Some(bidders) = &slot.prebid_bidders { + out.push_str("[creative_opportunities.slot.providers.prebid]\n"); + let rendered = bidders + .iter() + .map(|(name, params)| format!("{} = {}", toml_key(name), toml_inline_value(params))) + .collect::>() + .join(", "); + if rendered.is_empty() { + out.push_str("bidders = {}\n"); + } else { + out.push_str(&format!("bidders = {{ {rendered} }}\n")); + } + } + } + out +} + +/// Quotes and escapes a string as a TOML basic string, including control chars. +fn toml_string(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + control if (control as u32) < 0x20 => { + out.push_str(&format!("\\u{:04X}", control as u32)); + } + other => out.push(other), + } + } + out.push('"'); + out +} + +/// Renders a TOML table key: bare when it is a valid bare key, else a quoted key. +fn toml_key(key: &str) -> String { + let is_bare = !key.is_empty() + && key + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'); + if is_bare { + key.to_string() + } else { + toml_string(key) + } +} + +/// Renders a JSON value as a compact inline TOML value (for prebid bidder params). +fn toml_inline_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Null => "{}".to_string(), + serde_json::Value::Bool(bool) => bool.to_string(), + serde_json::Value::Number(number) => number.to_string(), + serde_json::Value::String(string) => toml_string(string), + serde_json::Value::Array(items) => { + let rendered = items + .iter() + .map(toml_inline_value) + .collect::>() + .join(", "); + format!("[{rendered}]") + } + serde_json::Value::Object(map) => { + let rendered = map + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_inline_value(value))) + .collect::>() + .join(", "); + format!("{{ {rendered} }}") + } + } +} + +/// Rewrites the `[creative_opportunities]` slot array of `existing` with the +/// pre-rendered `rendered_slots` text, updating `gam_network_id` and preserving +/// all other sections and comments. +/// +/// If the config has no `[creative_opportunities]` section, a fresh one is +/// appended so `generate` works against a config that omits it. +fn splice_creative_slots( + existing: &str, + network_id: Option<&str>, + rendered_slots: &str, +) -> CliResult { + let rendered = rendered_slots.trim_matches('\n'); + + // No section yet — append a fresh one with the network id and slots. + if !existing + .lines() + .any(|line| line.trim() == "[creative_opportunities]") + { + let mut result = existing.to_string(); + if !result.is_empty() && !result.ends_with('\n') { + result.push('\n'); + } + result.push_str("\n[creative_opportunities]\n"); + if let Some(network_id) = network_id { + result.push_str(&format!("gam_network_id = \"{network_id}\"\n")); + } + result.push_str(rendered); + result.push('\n'); + return Ok(result); + } + + // Section exists — update `gam_network_id` (best-effort) and replace slots. + let mut document = existing.to_string(); + if let Some(network_id) = network_id { + if let Ok(updated) = replace_key_in_section( + &document, + "creative_opportunities", + "gam_network_id", + &format!("gam_network_id = \"{network_id}\""), + ) { + document = updated; + } + } + + let lines: Vec<&str> = document.lines().collect(); + let header = lines + .iter() + .position(|line| line.trim() == "[creative_opportunities]") + .ok_or_else(|| { + report_error("target config has no [creative_opportunities] section to update") + })?; + + let is_slot_table = |line: &str| { + let trimmed = line.trim_start(); + trimmed.starts_with("[[creative_opportunities.slot]]") + || trimmed.starts_with("[creative_opportunities.slot.") + }; + let is_unrelated_table = |line: &str| { + let trimmed = line.trim_start(); + trimmed.starts_with('[') && !is_slot_table(line) && trimmed != "[creative_opportunities]" + }; + + // Where the existing slot array begins (first slot table after the header), + // else the end of the scalar block (first unrelated table, or EOF). + let existing_start = lines[header + 1..] + .iter() + .position(|line| is_slot_table(line)) + .map(|offset| header + 1 + offset); + let start = existing_start.unwrap_or_else(|| { + lines[header + 1..] + .iter() + .position(|line| is_unrelated_table(line)) + .map_or(lines.len(), |offset| header + 1 + offset) + }); + // Where the slot array ends: first unrelated top-level table, or EOF. + let end = lines[start..] + .iter() + .position(|line| is_unrelated_table(line)) + .map_or(lines.len(), |offset| start + offset); + + let mut result = lines[..start].join("\n"); + if !result.is_empty() { + result.push('\n'); + } + result.push_str(rendered); + result.push('\n'); + let tail = lines[end..].join("\n"); + if !tail.is_empty() { + result.push('\n'); + result.push_str(&tail); + } + if existing.ends_with('\n') && !result.ends_with('\n') { + result.push('\n'); + } + Ok(result) +} + fn replace_key_in_section( document: &str, section: &str, @@ -432,7 +976,11 @@ mod tests { } impl AuditCollector for FakeCollector { - fn collect_page(&self, _target_url: &Url) -> CliResult { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { self.calls.set(self.calls.get() + 1); Ok(self.collected.clone()) } @@ -458,6 +1006,7 @@ mod tests { url: "https://cdn.publisher.example/app.js".to_string(), resource_type: Some("script".to_string()), }], + gpt_slots: Vec::new(), warnings: Vec::new(), } } @@ -470,6 +1019,7 @@ mod tests { no_js_assets: false, no_config: false, force: false, + cookies: Vec::new(), } } @@ -552,6 +1102,7 @@ mod tests { no_js_assets: false, no_config: false, force: false, + cookies: Vec::new(), }; let collector = FakeCollector::new(collected_page()); let mut out = Vec::new(); @@ -675,7 +1226,8 @@ mod tests { warnings: Vec::new(), }; - let draft = build_draft_config(&url, &artifact).expect("should build draft config"); + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); assert!(draft.contains("domain = \"www.publisher.example\"")); assert!(draft.contains("cookie_domain = \".www.publisher.example\"")); @@ -703,9 +1255,316 @@ mod tests { warnings: Vec::new(), }; - let draft = build_draft_config(&url, &artifact).expect("should build draft config"); + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); assert!(draft.contains("[integrations.google_tag_manager]\nenabled = false")); assert!(draft.contains("Detected google_tag_manager")); } + + #[test] + fn build_audit_outputs_reconstructs_creative_opportunity_slots() { + let collected = CollectedPage { + requested_url: "https://example.com/".to_string(), + final_url: "https://example.com/".to_string(), + page_title: Some("Example Publisher".to_string()), + html: "".to_string(), + script_tags: Vec::new(), + network_requests: vec![CollectedRequest { + url: "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C620x366\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid" + .to_string(), + resource_type: Some("fetch".to_string()), + }], + gpt_slots: Vec::new(), + warnings: Vec::new(), + }; + + let outputs = build_audit_outputs(&collected).expect("should build outputs"); + assert_eq!(outputs.ad_slot_count, 1, "should discover one slot"); + + // The drafted config must be valid TOML with the reconstructed slot. + let value = + toml::from_str::(&outputs.draft_config_toml).expect("draft parses"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("123456789")); + let slot = &creative["slot"][0]; + assert_eq!(slot["id"].as_str(), Some("leaderboard-1")); + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/123456789/desktop/homepage/leaderboard1") + ); + assert_eq!( + slot["formats"][0]["width"].as_integer(), + Some(970), + "should keep the 970x250 pixel size" + ); + assert!( + slot["providers"]["prebid"].is_table(), + "prev_scp test=prebid should emit a prebid provider" + ); + } + + fn discovered_header_slot() -> gpt_slots::DiscoveredSlots { + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + gpt_slots::discover_gpt_slots(®istry, &[], false) + } + + /// Rendered slot text for the discovered header slot, patterns = `/`. + fn header_rendered() -> String { + let merged = merge_slots(None, &discovered_header_slot(), &["/".to_string()], true); + render_slots(&merged) + } + + fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { + toml::from_str::(toml_str).expect("valid creative config") + } + + #[test] + fn splice_replaces_slots_and_preserves_other_sections() { + let existing = "[publisher]\ndomain = \"x\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\nprice_granularity = \"dense\"\n\n\ + [[creative_opportunities.slot]]\nid = \"old\"\ndiv_id = \"old\"\n\ + gam_unit_path = \"/111/old\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + assert!( + out.contains("gam_network_id = \"222\""), + "network id updated" + ); + assert!(!out.contains("id = \"old\""), "old slot removed"); + assert!( + out.contains("gam_unit_path = \"/222/homepage/header\""), + "new slot written" + ); + assert!( + out.contains("[publisher]") && out.contains("domain = \"x\""), + "publisher section preserved" + ); + assert!( + out.contains("[auction]") && out.contains("enabled = true"), + "trailing auction section preserved" + ); + toml::from_str::(&out).expect("spliced config is valid TOML"); + } + + #[test] + fn splice_creates_section_when_absent() { + // Config with no [creative_opportunities] at all — generate should append it. + let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "appended section carries the discovered network id" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert!( + value["publisher"]["domain"].as_str() == Some("x") + && value["auction"]["enabled"].as_bool() == Some(true), + "existing sections preserved when appending" + ); + } + + #[test] + fn splice_inserts_when_no_existing_slots() { + let existing = + "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header"), + "inserted slot id strips the div-gpt-ad- prefix" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "div_id keeps the stable stem" + ); + assert!( + value["auction"]["enabled"].as_bool() == Some(true), + "auction section preserved after inserted slots" + ); + } + + #[test] + fn merge_second_run_unions_page_patterns() { + // Existing slot on "/"; re-discovered this run with "/news/*". + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/news/*".to_string()], + false, + ); + + assert_eq!(merged.len(), 1, "same slot is not duplicated"); + assert_eq!( + merged[0].page_patterns, + vec!["/".to_string(), "/news/*".to_string()], + "this run's pattern is unioned into the existing slot" + ); + } + + #[test] + fn merge_keeps_existing_only_slots() { + // Existing has header + sidebar; this run re-sees only header. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 0.5\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header", "sidebar"], "sidebar preserved"); + let sidebar = merged + .iter() + .find(|slot| slot.id == "sidebar") + .expect("sidebar"); + assert_eq!( + sidebar.floor_price, + Some(0.5), + "hand-tuned fields preserved" + ); + } + + #[test] + fn merge_replace_wipes_existing() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + true, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header"], "--replace keeps only discovered slots"); + } + + #[test] + fn default_page_pattern_uses_path_or_root() { + assert_eq!( + default_page_pattern(&Url::parse("https://x/news/story").expect("url")), + "/news/story" + ); + assert_eq!( + default_page_pattern(&Url::parse("https://x/").expect("url")), + "/" + ); + } + + #[test] + fn resolve_network_id_prefers_discovered_unless_preserving_existing() { + let with_slots = existing_config( + "gam_network_id = \"111\"\n\n[[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\n\ + gam_unit_path = \"/111/s\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let empty = existing_config("gam_network_id = \"111\"\n"); + + // Real merge → keep existing. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), false).as_deref(), + Some("111") + ); + // Placeholder section with no slots → discovered wins. + assert_eq!( + resolve_network_id(Some(&empty), Some("222"), false).as_deref(), + Some("222") + ); + // --replace → discovered wins. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), true).as_deref(), + Some("222") + ); + // No existing config → discovered. + assert_eq!( + resolve_network_id(None, Some("222"), false).as_deref(), + Some("222") + ); + } + + #[test] + fn toml_key_quotes_only_non_bare_keys() { + assert_eq!(toml_key("zone"), "zone"); + assert_eq!(toml_key("ad-loc"), "ad-loc"); + assert_eq!(toml_key("a.b"), "\"a.b\""); + assert_eq!(toml_key("with space"), "\"with space\""); + assert_eq!(toml_key(""), "\"\""); + } + + #[test] + fn toml_string_escapes_quotes_backslashes_and_controls() { + assert_eq!(toml_string("a\"b\\c"), "\"a\\\"b\\\\c\""); + assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); + } + + #[test] + fn render_quotes_exotic_targeting_keys_to_valid_toml() { + let existing = existing_config( + "gam_network_id = \"1\"\n\n\ + [[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\ngam_unit_path = \"/1/s\"\n\ + page_patterns = [\"/\"]\nformats = [{ width = 300, height = 250 }]\n\ + targeting = { \"a.b\" = \"x\" }\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + let doc = format!( + "[creative_opportunities]\ngam_network_id = \"1\"\n{}", + render_slots(&merged) + ); + + toml::from_str::(&doc).expect("exotic targeting key renders as valid TOML"); + } } diff --git a/crates/trusted-server-cli/src/audit/mod.rs b/crates/trusted-server-cli/src/audit/mod.rs index 3bcbd6924..ba1378ca5 100644 --- a/crates/trusted-server-cli/src/audit/mod.rs +++ b/crates/trusted-server-cli/src/audit/mod.rs @@ -32,6 +32,24 @@ pub(crate) fn parse_http_url(raw: &str) -> Result { } } +/// Parses a `name=value` cookie argument into its `(name, value)` parts. +/// +/// Splits on the first `=` so cookie values may themselves contain `=`. The name +/// must be non-empty; the value may be empty. +/// +/// # Errors +/// +/// Returns a user-facing string when the input has no `=` or an empty name. +pub(crate) fn parse_cookie(raw: &str) -> Result<(String, String), String> { + let (name, value) = raw + .split_once('=') + .ok_or_else(|| format!("invalid cookie `{raw}` (expected NAME=VALUE)"))?; + if name.is_empty() { + return Err(format!("invalid cookie `{raw}` (empty name)")); + } + Ok((name.to_string(), value.to_string())) +} + /// `ts audit` arguments: an optional subcommand plus a hidden legacy URL positional. #[derive(Debug, Args)] pub(crate) struct AuditArgs { @@ -57,10 +75,39 @@ pub(crate) enum AuditSubcommand { /// `ts audit ad-templates` subcommands. #[derive(Debug, Subcommand)] pub(crate) enum AuditAdTemplatesCommand { + /// Scrape a live page's GPT slots and update the config's + /// `[creative_opportunities]` slots in place. + Generate(AuditAdTemplatesGenerateArgs), /// Verify ad-template slots for one or more live URLs. Verify(AuditAdTemplatesVerifyArgs), } +/// Arguments for `ts audit ad-templates generate `. +#[derive(Debug, Args)] +pub(crate) struct AuditAdTemplatesGenerateArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page URL to scrape for GPT slots (http or https). + #[arg(value_parser = parse_http_url)] + pub url: url::Url, + /// Glob applied to every slot discovered this run (e.g. `/`, `/news/*`). + /// Repeatable. Defaults to the scraped URL's path. Re-running with a + /// different pattern unions it into slots already in the config. + #[arg(long = "page-pattern", value_name = "GLOB")] + pub page_patterns: Vec, + /// Replace all existing slots instead of merging this run into them. + #[arg(long)] + pub replace: bool, + /// Preview the updated config on stdout instead of writing it. + #[arg(long)] + pub dry_run: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, +} + /// Arguments for `ts audit ad-templates verify ...`. #[derive(Debug, Args)] pub(crate) struct AuditAdTemplatesVerifyArgs { @@ -78,6 +125,11 @@ pub(crate) struct AuditAdTemplatesVerifyArgs { /// Perform a deterministic scroll pass after the initial settle. #[arg(long)] pub scroll: bool, + /// Cookie to send with each page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, #[command(flatten)] pub browser: BrowserOpts, } @@ -94,6 +146,23 @@ pub(crate) struct AuditAdTemplatesVerifyArgs { pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { match &args.command { Some(AuditSubcommand::Page(page_args)) => page::run_page(page_args), + Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { + let loaded = crate::app_config::load_settings(&gen_args.config)?; + let collector = generate::browser_collector::BrowserAuditCollector; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + generate::run_update_slots( + gen_args.url.as_str(), + &loaded.app_config_path, + loaded.settings.creative_opportunities.as_ref(), + &gen_args.page_patterns, + gen_args.replace, + &gen_args.cookies, + gen_args.dry_run, + &collector, + &mut out, + ) + } Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Verify(verify_args))) => { ad_templates::run_verify(verify_args) } @@ -109,3 +178,40 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { }, } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_cookie_splits_on_first_equals() { + let (name, value) = parse_cookie("datadome=abc=def~ghi").expect("should parse cookie"); + assert_eq!(name, "datadome", "name should be the pre-`=` portion"); + assert_eq!( + value, "abc=def~ghi", + "value should keep later `=` characters" + ); + } + + #[test] + fn parse_cookie_allows_empty_value() { + let (name, value) = parse_cookie("session=").expect("should parse empty value"); + assert_eq!(name, "session"); + assert!(value.is_empty(), "empty value should be allowed"); + } + + #[test] + fn parse_cookie_rejects_missing_equals() { + let err = parse_cookie("datadome").expect_err("should reject missing `=`"); + assert!( + err.contains("NAME=VALUE"), + "error should show expected form" + ); + } + + #[test] + fn parse_cookie_rejects_empty_name() { + let err = parse_cookie("=value").expect_err("should reject empty name"); + assert!(err.contains("empty name"), "error should name the problem"); + } +} diff --git a/crates/trusted-server-cli/src/audit/page.rs b/crates/trusted-server-cli/src/audit/page.rs index 648a09f5d..cda9970dd 100644 --- a/crates/trusted-server-cli/src/audit/page.rs +++ b/crates/trusted-server-cli/src/audit/page.rs @@ -53,6 +53,7 @@ fn run_with_collector( init_scripts: Vec::new(), scroll, collect_ad_evidence: false, + cookies: Vec::new(), })?; let stdout = io::stdout(); From b3374ae5972caca8e24900671c0d685e6484da3f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 7 Jul 2026 13:22:26 +0530 Subject: [PATCH 135/395] Apply optional follow-ups from server-side ad-template review - Roll SPA page-bids navigation `currentPath` back to the last applied path instead of the immediately-previous one, so an aborted-then-failed navigation can no longer strand a route behind the no-op guard; add a regression test. - Add a concurrent render-bridge test: two same-adId messages before the cache fetch resolves must collapse to one fetch (in-flight gate), two beacons. - Assert `OPTIONS /__ts/page-bids` is denied with 403 on every adapter (Axum/Cloudflare/Spin) in cross-adapter parity. - Add a `u32::MAX` banner-format test covering the imp-drop branch when all formats exceed `i32::MAX`. - Dedup the page-bids GET 403 into `page_bids_preflight_denied()`. - Fix stale comments/docs: `buffer_publisher_response_async`, soften the oversized-body comment, and correct the `firedBeacons` key doc. --- .../trusted-server-adapter-fastly/src/app.rs | 2 +- .../src/auction/endpoints.rs | 2 +- .../src/integrations/prebid.rs | 32 +++++++++ crates/trusted-server-core/src/publisher.rs | 14 ++-- .../tests/parity.rs | 68 +++++++++++++++++++ .../trusted-server-js/lib/src/core/types.ts | 2 +- .../lib/src/integrations/gpt/index.ts | 14 +++- .../lib/test/integrations/gpt/ad_init.test.ts | 53 +++++++++++++++ .../test/integrations/gpt/spa_hook.test.ts | 53 +++++++++++++++ 9 files changed, 225 insertions(+), 15 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 27fb18918..5321c32cd 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -718,7 +718,7 @@ async fn dispatch_fallback( } else if state.registry.has_route(&method, &path) { // Integration-proxy responses are not bounded by publisher.max_buffered_body_bytes. // Only the handle_publisher_request branch below routes through - // buffer_publisher_response. Integration responses are small in practice + // buffer_publisher_response_async. Integration responses are small in practice // and the EdgeZero flag is off by default; extend the cap here if that changes. state .registry diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index c642c9ec3..56c104ab6 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -110,7 +110,7 @@ pub async fn handle_auction( services: &RuntimeServices, req: Request, ) -> Result, Report> { - // Reject oversized bodies before any allocation. The Content-Length + // Reject oversized bodies before core buffers/parses them. The Content-Length // pre-check stops well-behaved clients early; the post-read check defends // against clients that lie about (or omit) the header. let content_length_exceeded = req diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index dd73dabae..596eb7d33 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -3472,6 +3472,38 @@ server_url = "https://prebid.example" assert_eq!(formats[0].h, Some(250), "should preserve valid height"); } + #[test] + fn to_openrtb_drops_imp_when_all_banner_formats_exceed_i32_max() { + // The build-time bound: every banner format's u32 dimensions pass through + // `to_openrtb_i32`, which omits any value above i32::MAX. When a slot's + // only format is out of range (here u32::MAX), no valid formats remain, so + // the whole imp must be dropped rather than emitted with an empty format + // list — a sizeless imp is unbiddable and would only waste an SSP call. + let provider = PrebidAuctionProvider::new(base_config()); + let mut auction_request = create_test_auction_request(); + auction_request.slots[0].formats = vec![AdFormat { + media_type: MediaType::Banner, + width: u32::MAX, + height: u32::MAX, + }]; + + let settings = make_settings(); + let request = build_test_request(); + let context = create_test_auction_context(&settings, &request); + + let openrtb = provider.to_openrtb( + &auction_request, + &context, + None, + make_request_info(&context), + ); + + assert!( + openrtb.imp.is_empty(), + "should drop the imp entirely when every banner format exceeds i32::MAX" + ); + } + #[test] fn to_openrtb_sets_site_ref_from_referer_header() { let provider = PrebidAuctionProvider::new(base_config()); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d9944ad95..7482de88a 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1915,8 +1915,10 @@ fn page_bids_request_allowed(req: &Request) -> bool { } } -/// Builds the `403 Forbidden` returned for a CORS preflight (`OPTIONS`) to the -/// side-effecting `/__ts/page-bids` endpoint. +/// Builds the `403 Forbidden` returned when the side-effecting +/// `/__ts/page-bids` endpoint refuses a request — both the CORS preflight +/// (`OPTIONS`) and the GET cross-site gate ([`page_bids_request_allowed`]) +/// return this single denial shape. /// /// The GET handler's [`page_bids_request_allowed`] gate trusts the /// `X-TSJS-Page-Bids` header precisely because this endpoint never grants a @@ -1985,13 +1987,7 @@ pub async fn handle_page_bids( .and_then(|v| v.to_str().ok()), req.headers().contains_key("x-tsjs-page-bids") ); - let mut response = Response::new(EdgeBody::from("Forbidden")); - *response.status_mut() = StatusCode::FORBIDDEN; - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); - return Ok(response); + return Ok(page_bids_preflight_denied()); } let path_param = req diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index a5f32b275..e85b1d8d1 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -265,6 +265,48 @@ async fn spin_authorized_json(method: &str, uri: &str, body: &str) -> (u16, Head (resp.status().as_u16(), resp.headers().clone()) } +/// Send an OPTIONS request to the Axum adapter and return (status, headers). +async fn axum_options(uri: &str) -> (u16, HeaderMap) { + let mut svc = EdgeZeroAxumService::new(axum_router()); + let req = AxumRequest::builder() + .method("OPTIONS") + .uri(uri) + .body(AxumBody::empty()) + .expect("should build OPTIONS request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + +/// Send an OPTIONS request to the Cloudflare adapter and return (status, headers). +async fn cf_options(uri: &str) -> (u16, HeaderMap) { + let router = cf_router(); + let req = request_builder() + .method("OPTIONS") + .uri(uri) + .body(edgezero_core::body::Body::empty()) + .expect("should build OPTIONS request"); + let resp = router.oneshot(req).await.expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + +/// Send an OPTIONS request to the Spin adapter and return (status, headers). +async fn spin_options(uri: &str) -> (u16, HeaderMap) { + let router = spin_router(); + let req = request_builder() + .method("OPTIONS") + .uri(uri) + .body(edgezero_core::body::Body::empty()) + .expect("should build OPTIONS request"); + let resp = router.oneshot(req).await.expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + // --------------------------------------------------------------------------- // Route parity: same route → same status on all adapters // --------------------------------------------------------------------------- @@ -652,6 +694,32 @@ async fn auction_not_challenged_by_auth_parity() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn page_bids_options_preflight_denied_parity() { + // OPTIONS /__ts/page-bids is a CORS preflight to a side-effecting endpoint. + // Every adapter must refuse it with 403 rather than proxy it to the origin: + // a permissive origin preflight would let a cross-site page defeat the GET + // handler's `X-TSJS-Page-Bids` gate and trigger real auctions in a visitor's + // browser. The denial is unconditional (independent of creative-opportunity + // configuration), so all adapters must agree on 403. + let (axum_status, _) = axum_options("/__ts/page-bids").await; + let (cf_status, _) = cf_options("/__ts/page-bids").await; + let (spin_status, _) = spin_options("/__ts/page-bids").await; + + assert_eq!( + axum_status, 403, + "Axum OPTIONS /__ts/page-bids must be denied with 403, got {axum_status}" + ); + assert_eq!( + cf_status, 403, + "Cloudflare OPTIONS /__ts/page-bids must be denied with 403, got {cf_status}" + ); + assert_eq!( + spin_status, 403, + "Spin OPTIONS /__ts/page-bids must be denied with 403, got {spin_status}" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn spin_auction_ignores_spoofed_forwarded_headers() { // POST /auction feeds prebid request signing via `RequestInfo::from_request`, diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index ec2882efb..360e2aa49 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -99,7 +99,7 @@ export interface TsjsApi { /** Maps actualDivId → slotId for slotRenderEnded billing lookup. */ divToSlotId?: Record; /** - * Win/billing beacons already fired, keyed by `slotId|bidIdentity`. + * Win/billing beacons already fired, keyed by `slotId|bidIdentity|kind|url`. * Used by the GPT render bridge so a bid's nurl/burl fire at most once even * across repeated Prebid Universal Creative requests for the same adId. */ diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 8b0b8d529..ca4689684 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -713,10 +713,15 @@ export function installSpaAuctionHook(): void { // can be called with the current URL, so guard every entry point against // re-requesting impressions for a path we already loaded. let currentPath = location.pathname; + // Last path whose slots/bids were actually applied — the initial SSR page + // counts. A failed navigation rolls `currentPath` back to this rather than to + // the immediately-previous committed value: on rapid A→B where A was aborted + // mid-flight and B then fails, rolling back to A (never loaded) would strand + // it behind the no-op guard, so we roll back to the last applied route instead. + let lastAppliedPath = location.pathname; async function onNavigate(path: string): Promise { if (path === currentPath) return; - const previousPath = currentPath; currentPath = path; inflight?.abort(); const controller = new AbortController(); @@ -737,7 +742,7 @@ export function installSpaAuctionHook(): void { // committed path back so a later navigation here retries instead of // being skipped by the no-op guard at the top. Only roll back when no // newer navigation has already advanced currentPath. - if (inflight === controller) currentPath = previousPath; + if (inflight === controller) currentPath = lastAppliedPath; return; } const data = (await res.json()) as PageBidsResponse; @@ -748,6 +753,9 @@ export function installSpaAuctionHook(): void { if (inflight !== controller) return; ts.adSlots = data.slots; ts.bids = data.bids; + // This route is now the committed, loaded state — a later failed + // navigation rolls back here, and a return trip no-ops correctly. + lastAppliedPath = path; // An empty page-bids response (auction kill switch or consent gate) carries // no TS slots. Only run adInit() when there are slots to apply or prior TS // state to sweep — otherwise a consent-denied or kill-switched navigation @@ -761,7 +769,7 @@ export function installSpaAuctionHook(): void { } } catch (err) { if (err instanceof DOMException && err.name === 'AbortError') return; - if (inflight === controller) currentPath = previousPath; + if (inflight === controller) currentPath = lastAppliedPath; log.warn('SPA auction hook: fetch failed', err); } } diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index b82542695..4a6368768 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -986,6 +986,59 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('fetches PBS Cache once when two same-adId messages race before the fetch resolves', async () => { + // Concurrent render double-fire guard: two 'Prebid Request' messages for the + // same adId can arrive before the first cache fetch settles. The in-flight + // `renderingAdIds` gate must collapse them to a single fetch — the persistent + // firedBeacons dedup only engages after a fetch resolves, so it cannot stop + // the second fetch on its own. Deferring the fetch keeps both messages in the + // window where only the in-flight gate can prevent the duplicate. + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const mockAd = '
Test Creative
'; + let resolveFetch: (value: Response) => void = () => {}; + fetchStub.mockReturnValue( + new Promise((resolve) => { + resolveFetch = resolve; + }) + ); + + const bridgeListener = await captureBridgeListener(); + + const stopSpy = vi.fn(); + const portMessages: string[] = []; + const fakePort = { postMessage: (s: string) => portMessages.push(s) }; + const source = createTrustedSlotIframe(); + + const dispatch = (): unknown => + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [fakePort], + source, + stopImmediatePropagation: stopSpy, + }) as unknown as MessageEvent + ); + + // Both messages dispatched before the deferred fetch resolves. + dispatch(); + dispatch(); + + // The second message hit the in-flight gate — only one fetch launched. + expect(fetchStub).toHaveBeenCalledTimes(1); + + // Resolve the single fetch and flush its .then chain. + resolveFetch({ ok: true, text: () => Promise.resolve(mockAd) } as Response); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(fetchStub).toHaveBeenCalledTimes(1); + expect(portMessages).toHaveLength(1); + // A single render still fires both win and billing beacons exactly once. + expect(beaconSpy).toHaveBeenCalledWith('https://ssp.example/win'); + expect(beaconSpy).toHaveBeenCalledWith('https://ssp.example/bill'); + expect(beaconSpy).toHaveBeenCalledTimes(2); + beaconSpy.mockRestore(); + }); + it('responds with adm without fetching PBS Cache when debug adm is available', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const debugAdm = '
Debug Creative
'; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 2ef3a1746..9a08defcb 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -353,6 +353,59 @@ describe('installSpaAuctionHook', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('does not strand a path that was aborted mid-flight then failed on the next nav', async () => { + // Rapid A→B where A is aborted mid-flight and B then fails must roll + // `currentPath` back to the last *applied* path (here the initial route), + // not to A. Rolling back to A — which never loaded — would leave it behind + // the no-op guard so a later real navigation to A never re-fetches. + document.body.innerHTML = '
'; + let resolveA: ((value: unknown) => void) | undefined; + fetchStub + // A: still in flight when B starts (aborted, never settles on its own). + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveA = resolve; + }) + ) + // B: fails. + .mockResolvedValueOnce({ ok: false, status: 500 }) + // A retried: succeeds. + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + slots: [{ id: 'a', div_id: 'div-a' }], + bids: { a: { hb_pb: '1.00' } }, + }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + // A starts (left in flight), then B aborts A and fails. + history.pushState({}, '', '/a'); + history.pushState({}, '', '/b'); + await flushAsync(); + expect(ts.adSlots).toBeUndefined(); + + // Navigate back to /a. With the rollback keyed to the last applied path + // (the initial route) instead of B's previous path (/a), this is NOT + // swallowed by the no-op guard and re-fetches. + history.pushState({}, '', '/a'); + await flushAsync(); + + expect(fetchStub).toHaveBeenCalledTimes(3); + expect(ts.adSlots).toEqual([{ id: 'a', div_id: 'div-a' }]); + expect(adInit).toHaveBeenCalledTimes(1); + + // The original aborted A fetch resolving late must not clobber the retry. + resolveA?.({ ok: true, json: async () => ({ slots: [{ id: 'stale' }], bids: {} }) }); + await flushAsync(); + expect(ts.adSlots).toEqual([{ id: 'a', div_id: 'div-a' }]); + }); + it('is idempotent — repeated install calls do not double-fetch a navigation', async () => { fetchStub.mockResolvedValue({ ok: true, From f444a15d03ba670d7bc24511a84858562080ebfd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 7 Jul 2026 21:57:09 +0530 Subject: [PATCH 136/395] Harden ad-template audit against hostile page data - Escape page-controlled slot fields and validate the gampad network id before splicing scraped values into trusted-server.toml - Add navigation and teardown timeouts to the verify browser collector - Snapshot evidence before the scroll pass so load-time entries keep phase initial_load; add a Chrome-gated regression fixture - Cap collector evidence lists in the injected script and after decode - Preserve CRLF line endings and render non-finite floor_price as valid TOML when updating configs in place - Cover all 128 gate combinations in the core ad-stack mirror test - Extract slot TOML rendering/merging/splicing into slot_toml.rs --- .../commands/audit/ad_template_collector.js | 27 +- .../src/commands/audit/browser.rs | 91 ++- .../src/commands/audit/generate/gpt_slots.rs | 23 +- .../src/commands/audit/generate/mod.rs | 714 +--------------- .../src/commands/audit/generate/slot_toml.rs | 762 ++++++++++++++++++ .../src/creative_opportunities.rs | 4 +- 6 files changed, 920 insertions(+), 701 deletions(-) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js index bc83772d8..c01074376 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -18,16 +18,24 @@ const __ts_ev = (window.__tsAdTemplateEvidence = window.__tsAdTemplateEvidence | const __ts_phase = () => (window.__tsScrollPhase ? "scroll" : "initial_load") +// Hard cap per evidence list so a hostile page cannot grow the store without +// bound; the page controls how many slots/elements/warnings it produces. +const __ts_max_entries = 1024 +function __ts_push(list, entry) { + if (list.length < __ts_max_entries) list.push(entry) +} + function __ts_normalize_sizes(sizes) { const out = [] if (!Array.isArray(sizes)) return out // Accept [w, h] or [[w, h], ...]; treat numeric-leading arrays as a single pair. const pairs = typeof sizes[0] === "number" ? [sizes] : sizes for (const size of pairs) { + if (out.length >= __ts_max_entries) break if (Array.isArray(size) && typeof size[0] === "number" && typeof size[1] === "number") { out.push([size[0], size[1]]) } else { - __ts_ev.warnings.push({ + __ts_push(__ts_ev.warnings, { code: "fluid_size_ignored", message: "non-numeric GPT size ignored", }) @@ -37,7 +45,7 @@ function __ts_normalize_sizes(sizes) { } function __ts_record_define_slot(adUnitPath, sizes, divId) { - __ts_ev.gpt_slots.push({ + __ts_push(__ts_ev.gpt_slots, { gam_unit_path: String(adUnitPath), div_id: String(divId), sizes: __ts_normalize_sizes(sizes), @@ -63,7 +71,7 @@ function __ts_wrap_googletag(googletag) { try { __ts_record_define_slot(adUnitPath, sizes, divId) } catch (error) { - __ts_ev.warnings.push({ code: "define_slot_capture_failed", message: String(error) }) + __ts_push(__ts_ev.warnings, { code: "define_slot_capture_failed", message: String(error) }) } return slot } @@ -80,14 +88,14 @@ function __ts_wrap_apstag(apstag) { try { const slots = (config && config.slots) || [] for (const slot of slots) { - __ts_ev.aps_calls.push({ + __ts_push(__ts_ev.aps_calls, { slot_id: String(slot.slotID || slot.slotName || ""), sizes: __ts_normalize_sizes(slot.sizes), phase: __ts_phase(), }) } } catch (error) { - __ts_ev.warnings.push({ code: "aps_capture_failed", message: String(error) }) + __ts_push(__ts_ev.warnings, { code: "aps_capture_failed", message: String(error) }) } return originalFetchBids.apply(this, arguments) } @@ -124,7 +132,7 @@ window.__tsCollectAdTemplateEvidence = function () { const id = element.id if (id.endsWith("-container")) continue if (__ts_prefixes.some((prefix) => id.startsWith(prefix)) && !seen.has(id)) { - __ts_ev.dom_ids.push({ dom_id: id, phase: __ts_phase() }) + __ts_push(__ts_ev.dom_ids, { dom_id: id, phase: __ts_phase() }) seen.add(id) } } @@ -139,6 +147,7 @@ window.__tsCollectAdTemplateEvidence = function () { const rawSizes = typeof slot.getSizes === "function" ? slot.getSizes() : [] const sizes = [] for (const size of rawSizes) { + if (sizes.length >= __ts_max_entries) break if (size && typeof size.getWidth === "function") { sizes.push([size.getWidth(), size.getHeight()]) } else if (Array.isArray(size) && typeof size[0] === "number") { @@ -149,7 +158,7 @@ window.__tsCollectAdTemplateEvidence = function () { (entry) => entry.gam_unit_path === String(path) && entry.div_id === String(divId) ) if (!exists) { - __ts_ev.gpt_slots.push({ + __ts_push(__ts_ev.gpt_slots, { gam_unit_path: String(path), div_id: String(divId), sizes, @@ -157,12 +166,12 @@ window.__tsCollectAdTemplateEvidence = function () { }) } } catch (error) { - __ts_ev.warnings.push({ code: "gpt_scrape_failed", message: String(error) }) + __ts_push(__ts_ev.warnings, { code: "gpt_scrape_failed", message: String(error) }) } } } } catch (error) { - __ts_ev.warnings.push({ code: "collect_failed", message: String(error) }) + __ts_push(__ts_ev.warnings, { code: "collect_failed", message: String(error) }) } return __ts_ev } diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs index e302b74d2..9f6aca1e6 100644 --- a/crates/trusted-server-cli/src/commands/audit/browser.rs +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -31,6 +31,13 @@ const CHROME_NAMES: &[&str] = &[ /// Poll interval while waiting for the page network to settle, in milliseconds. const SETTLE_POLL_MS: u64 = 250; +/// Hard cap on page navigation so a stalled load cannot hang the audit. +const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); +/// Hard cap per decoded evidence list, mirroring the collector script's +/// `__ts_max_entries`, so a hostile page cannot inflate CLI memory. +const MAX_EVIDENCE_ENTRIES: usize = 1024; +/// Hard cap on browser teardown so a wedged Chrome cannot hang the audit. +const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); /// Default quiet window (no new resources) marking the page settled. const DEFAULT_SETTLE_QUIET_MS: u64 = 750; /// Default hard cap on settling so slow/ad-heavy pages still terminate. @@ -228,9 +235,10 @@ async fn collect( let result = collect_with_browser(&browser, request, settle_config).await; - // Best-effort teardown; ignore errors since we already have a result. - let _ = browser.close().await; - let _ = browser.wait().await; + // Best-effort teardown; ignore errors since we already have a result, but + // bound it so a Chrome that ignores `close` cannot hang the command. + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.close()).await; + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.wait()).await; handler_task.abort(); result @@ -267,16 +275,29 @@ async fn collect_with_browser( .map_err(|error| format!("failed to set cookie `{name}`: {error}"))?; } - page.goto(request.url.as_str()) + tokio::time::timeout(NAVIGATION_TIMEOUT, page.goto(request.url.as_str())) .await + .map_err(|_| format!("navigation to {} timed out", request.url))? .map_err(|error| format!("failed to navigate to {}: {error}", request.url))?; - page.wait_for_navigation() + tokio::time::timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation()) .await + .map_err(|_| format!("navigation to {} timed out", request.url))? .map_err(|error| format!("failed to read main document navigation response: {error}"))?; settle(&page, settle_config).await; if request.scroll { + if request.collect_ad_evidence { + // Snapshot evidence before scrolling so entries already present at + // initial load keep phase "load"; the store dedups first-seen, so + // the post-scroll scrape only adds genuinely scroll-phase entries. + let _ = page + .evaluate( + "(typeof window.__tsCollectAdTemplateEvidence === 'function' \ + && window.__tsCollectAdTemplateEvidence(), null)", + ) + .await; + } scroll_page(&page).await; settle(&page, settle_config).await; } @@ -386,7 +407,15 @@ async fn extract_ad_evidence( None } Some(value) => match serde_json::from_value::(value) { - Ok(evidence) => Some(evidence), + Ok(mut evidence) => { + // Defense in depth: the injected script caps these lists, but the + // page owns that store, so re-cap after decode. + evidence.dom_ids.truncate(MAX_EVIDENCE_ENTRIES); + evidence.gpt_slots.truncate(MAX_EVIDENCE_ENTRIES); + evidence.aps_calls.truncate(MAX_EVIDENCE_ENTRIES); + evidence.warnings.truncate(MAX_EVIDENCE_ENTRIES); + Some(evidence) + } Err(error) => { warnings.push(Warning { code: "ad_evidence_decode_failed".to_string(), @@ -505,4 +534,54 @@ mod tests { "should capture the configured-prefix DOM id" ); } + + #[test] + fn scroll_pass_keeps_initial_load_phase_for_load_time_evidence() { + if !chrome_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + aps_slot_ids: Vec::new(), + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: true, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + // The slot and DOM id exist at load time, so the pre-scroll snapshot + // must record them as initial-load even though a scroll pass ran. + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0" + && dom.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad), + "load-time DOM id should keep phase initial_load under --scroll" + ); + assert!( + evidence.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/news/atf" + && slot.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad + }), + "load-time GPT slot should keep phase initial_load under --scroll" + ); + } } diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index 14172ff0c..b7d595dbc 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -210,7 +210,13 @@ fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot)> { let iu_parts = iu_parts?; let mut parts = iu_parts.split(',').filter(|part| !part.is_empty()); - let network_id = parts.next()?.to_string(); + // Mirror the registry path's validation: a GAM network id is digits only. + // The percent-decoded query value is page-controlled and gets spliced into + // generated TOML, so reject anything else. + let network_id = parts + .next() + .filter(|segment| segment.bytes().all(|byte| byte.is_ascii_digit()))? + .to_string(); let gam_unit_path = format!("/{}", iu_parts.replace(',', "/")); // A usable unit path needs the network id plus at least one path segment. parts.next()?; @@ -381,6 +387,21 @@ mod tests { ); } + #[test] + fn skips_requests_with_non_numeric_network_id() { + // A page-controlled iu_parts value must not smuggle a non-numeric + // network id (it gets spliced into generated TOML). + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%22evil%2Cslot&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a non-numeric network id should be rejected" + ); + assert_eq!(discovered.gam_network_id, None); + } + #[test] fn falls_back_to_pb_szs_when_prev_iu_szs_absent() { let discovered = from_requests(&[request( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index bf1262697..a6f2546e6 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -2,20 +2,22 @@ mod analyzer; pub(crate) mod browser_collector; pub(crate) mod collector; mod gpt_slots; +mod slot_toml; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; use serde::Serialize; -use trusted_server_core::auction::types::MediaType; -use trusted_server_core::creative_opportunities::{ - CreativeOpportunitiesConfig, CreativeOpportunitySlot, -}; +use trusted_server_core::creative_opportunities::CreativeOpportunitiesConfig; use url::Url; use crate::commands::audit::generate::collector::AuditCollector; +use crate::commands::audit::generate::slot_toml::{ + merge_slots, render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, + toml_string, +}; use crate::commands::config::init::EXAMPLE_CONFIG; use crate::error::{CliResult, cli_error, report_error}; @@ -386,7 +388,7 @@ fn build_draft_config( &draft, "creative_opportunities", "gam_network_id", - &format!("gam_network_id = \"{network_id}\""), + &format!("gam_network_id = {}", toml_string(network_id)), )?; } draft.push_str(&render_discovered_slots(target_url, slots)); @@ -414,14 +416,15 @@ fn render_discovered_slots(target_url: &Url, slots: &gpt_slots::DiscoveredSlots) .join(", "); out.push_str(&format!( "\n[[creative_opportunities.slot]]\n\ - id = \"{id}\"\n\ - div_id = \"{div_id}\"\n\ - gam_unit_path = \"{gam_unit_path}\"\n\ - page_patterns = [\"{page_pattern}\"]\n\ + id = {id}\n\ + div_id = {div_id}\n\ + gam_unit_path = {gam_unit_path}\n\ + page_patterns = [{page_pattern}]\n\ formats = [{formats}]\n", - id = slot.id, - div_id = slot.div_id, - gam_unit_path = slot.gam_unit_path, + id = toml_string(&slot.id), + div_id = toml_string(&slot.div_id), + gam_unit_path = toml_string(&slot.gam_unit_path), + page_pattern = toml_string(page_pattern), )); if slot.has_prebid { out.push_str("[creative_opportunities.slot.providers.prebid]\nbidders = {}\n"); @@ -511,29 +514,6 @@ pub(crate) fn run_update_slots( ) .map_err(|error| report_error(format!("failed to write command output: {error}"))) } - -/// Chooses the `gam_network_id` to write. -/// -/// The existing id is kept only when a real merge preserves existing slots. -/// On `--replace`, or when the config had no slots (e.g. a placeholder -/// `[creative_opportunities]` section), the discovered id wins — mirroring -/// [`merge_slots`], which returns discovered-only in those cases. -fn resolve_network_id( - existing: Option<&CreativeOpportunitiesConfig>, - discovered_network_id: Option<&str>, - replace: bool, -) -> Option { - let existing_network_id = existing.map(|config| config.gam_network_id.clone()); - let preserving_existing = !replace && existing.is_some_and(|config| !config.slot.is_empty()); - if preserving_existing { - existing_network_id.or_else(|| discovered_network_id.map(str::to_string)) - } else { - discovered_network_id - .map(str::to_string) - .or(existing_network_id) - } -} - /// The default page pattern for a scraped URL: its path, or `/` for the root. fn default_page_pattern(target_url: &Url) -> String { let path = target_url.path(); @@ -544,414 +524,6 @@ fn default_page_pattern(target_url: &Url) -> String { } } -/// A slot ready to render — the union of discovered and existing fields, without -/// the core type's `pub(crate)` compiled-pattern cache. -#[derive(Debug, Clone)] -struct RenderSlot { - id: String, - div_id: Option, - gam_unit_path: Option, - page_patterns: Vec, - /// `(width, height, non-banner media type)`. - formats: Vec<(u32, u32, Option<&'static str>)>, - floor_price: Option, - targeting: BTreeMap, - aps_slot_id: Option, - /// `Some` when the slot runs Prebid; the map is per-bidder params (often empty). - prebid_bidders: Option>, -} - -impl RenderSlot { - /// The stable identity used to match slots across runs: the div id (or slot - /// id), with any trailing `-` trimmed so hand-authored stems still match. - fn key(&self) -> String { - self.div_id - .as_deref() - .unwrap_or(&self.id) - .trim_end_matches('-') - .to_string() - } - - fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { - Self { - id: slot.id.clone(), - div_id: Some(slot.div_id.clone()), - gam_unit_path: Some(slot.gam_unit_path.clone()), - page_patterns: patterns.to_vec(), - formats: slot - .formats - .iter() - .map(|&(width, height)| (width, height, None)) - .collect(), - floor_price: None, - targeting: BTreeMap::new(), - aps_slot_id: None, - prebid_bidders: slot.has_prebid.then(BTreeMap::new), - } - } - - fn from_existing(slot: &CreativeOpportunitySlot) -> Self { - Self { - id: slot.id.clone(), - div_id: slot.div_id.clone(), - gam_unit_path: slot.gam_unit_path.clone(), - page_patterns: slot.page_patterns.clone(), - formats: slot - .formats - .iter() - .map(|format| { - ( - format.width, - format.height, - media_type_label(&format.media_type), - ) - }) - .collect(), - floor_price: slot.floor_price, - targeting: slot - .targeting - .iter() - .map(|(key, value)| (key.clone(), value.clone())) - .collect(), - aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), - prebid_bidders: slot.providers.prebid.as_ref().map(|prebid| { - prebid - .bidders - .iter() - .map(|(name, params)| (name.clone(), params.clone())) - .collect() - }), - } - } -} - -/// The non-default (non-banner) media-type label to emit, or `None` for banner. -fn media_type_label(media_type: &MediaType) -> Option<&'static str> { - match media_type { - MediaType::Banner => None, - MediaType::Video => Some("video"), - MediaType::Native => Some("native"), - } -} - -/// Merges discovered slots into the existing slot set, keyed by [`RenderSlot::key`]. -/// -/// - `--replace` (or no existing slots): the result is exactly the discovered set. -/// - Otherwise existing slots are preserved (covering other pages / hand-tuned -/// fields); a slot re-seen this run has `run_patterns` unioned into its -/// `page_patterns`; slots seen only this run are appended. -fn merge_slots( - existing: Option<&CreativeOpportunitiesConfig>, - discovered: &gpt_slots::DiscoveredSlots, - run_patterns: &[String], - replace: bool, -) -> Vec { - let discovered_slots: Vec = discovered - .slots - .iter() - .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) - .collect(); - - let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); - if replace || existing_slots.is_empty() { - return discovered_slots; - } - - let mut merged: Vec = existing_slots - .iter() - .map(RenderSlot::from_existing) - .collect(); - for slot in discovered_slots { - let key = slot.key(); - if let Some(present) = merged.iter_mut().find(|existing| existing.key() == key) { - for pattern in &slot.page_patterns { - if !present.page_patterns.contains(pattern) { - present.page_patterns.push(pattern.clone()); - } - } - } else { - merged.push(slot); - } - } - merged -} - -/// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. -fn render_slots(slots: &[RenderSlot]) -> String { - let mut out = String::from( - "\n# Slots managed by `ts audit ad-templates generate`.\n\ - # Review page_patterns and formats before validating/pushing.\n", - ); - for slot in slots { - out.push_str("\n[[creative_opportunities.slot]]\n"); - out.push_str(&format!("id = {}\n", toml_string(&slot.id))); - if let Some(div_id) = &slot.div_id { - out.push_str(&format!("div_id = {}\n", toml_string(div_id))); - } - if let Some(path) = &slot.gam_unit_path { - out.push_str(&format!("gam_unit_path = {}\n", toml_string(path))); - } - let patterns = slot - .page_patterns - .iter() - .map(|pattern| toml_string(pattern)) - .collect::>() - .join(", "); - out.push_str(&format!("page_patterns = [{patterns}]\n")); - let formats = slot - .formats - .iter() - .map(|(width, height, media_type)| match media_type { - Some(kind) => { - format!("{{ width = {width}, height = {height}, media_type = \"{kind}\" }}") - } - None => format!("{{ width = {width}, height = {height} }}"), - }) - .collect::>() - .join(", "); - out.push_str(&format!("formats = [{formats}]\n")); - if let Some(floor) = slot.floor_price { - out.push_str(&format!("floor_price = {floor}\n")); - } - if !slot.targeting.is_empty() { - let pairs = slot - .targeting - .iter() - .map(|(key, value)| format!("{} = {}", toml_key(key), toml_string(value))) - .collect::>() - .join(", "); - out.push_str(&format!("targeting = {{ {pairs} }}\n")); - } - if let Some(slot_id) = &slot.aps_slot_id { - out.push_str("[creative_opportunities.slot.providers.aps]\n"); - out.push_str(&format!("slot_id = {}\n", toml_string(slot_id))); - } - if let Some(bidders) = &slot.prebid_bidders { - out.push_str("[creative_opportunities.slot.providers.prebid]\n"); - let rendered = bidders - .iter() - .map(|(name, params)| format!("{} = {}", toml_key(name), toml_inline_value(params))) - .collect::>() - .join(", "); - if rendered.is_empty() { - out.push_str("bidders = {}\n"); - } else { - out.push_str(&format!("bidders = {{ {rendered} }}\n")); - } - } - } - out -} - -/// Quotes and escapes a string as a TOML basic string, including control chars. -fn toml_string(value: &str) -> String { - let mut out = String::with_capacity(value.len() + 2); - out.push('"'); - for ch in value.chars() { - match ch { - '"' => out.push_str("\\\""), - '\\' => out.push_str("\\\\"), - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - '\t' => out.push_str("\\t"), - control if (control as u32) < 0x20 => { - out.push_str(&format!("\\u{:04X}", control as u32)); - } - other => out.push(other), - } - } - out.push('"'); - out -} - -/// Renders a TOML table key: bare when it is a valid bare key, else a quoted key. -fn toml_key(key: &str) -> String { - let is_bare = !key.is_empty() - && key - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'); - if is_bare { - key.to_string() - } else { - toml_string(key) - } -} - -/// Renders a JSON value as a compact inline TOML value (for prebid bidder params). -fn toml_inline_value(value: &serde_json::Value) -> String { - match value { - serde_json::Value::Null => "{}".to_string(), - serde_json::Value::Bool(bool) => bool.to_string(), - serde_json::Value::Number(number) => number.to_string(), - serde_json::Value::String(string) => toml_string(string), - serde_json::Value::Array(items) => { - let rendered = items - .iter() - .map(toml_inline_value) - .collect::>() - .join(", "); - format!("[{rendered}]") - } - serde_json::Value::Object(map) => { - let rendered = map - .iter() - .map(|(key, value)| format!("{} = {}", toml_key(key), toml_inline_value(value))) - .collect::>() - .join(", "); - format!("{{ {rendered} }}") - } - } -} - -/// Rewrites the `[creative_opportunities]` slot array of `existing` with the -/// pre-rendered `rendered_slots` text, updating `gam_network_id` and preserving -/// all other sections and comments. -/// -/// If the config has no `[creative_opportunities]` section, a fresh one is -/// appended so `generate` works against a config that omits it. -fn splice_creative_slots( - existing: &str, - network_id: Option<&str>, - rendered_slots: &str, -) -> CliResult { - let rendered = rendered_slots.trim_matches('\n'); - - // No section yet — append a fresh one with the network id and slots. - if !existing - .lines() - .any(|line| line.trim() == "[creative_opportunities]") - { - let mut result = existing.to_string(); - if !result.is_empty() && !result.ends_with('\n') { - result.push('\n'); - } - result.push_str("\n[creative_opportunities]\n"); - if let Some(network_id) = network_id { - result.push_str(&format!("gam_network_id = \"{network_id}\"\n")); - } - result.push_str(rendered); - result.push('\n'); - return Ok(result); - } - - // Section exists — update `gam_network_id` (best-effort) and replace slots. - let mut document = existing.to_string(); - if let Some(network_id) = network_id - && let Ok(updated) = replace_key_in_section( - &document, - "creative_opportunities", - "gam_network_id", - &format!("gam_network_id = \"{network_id}\""), - ) - { - document = updated; - } - - let lines: Vec<&str> = document.lines().collect(); - let header = lines - .iter() - .position(|line| line.trim() == "[creative_opportunities]") - .ok_or_else(|| { - report_error("target config has no [creative_opportunities] section to update") - })?; - - let is_slot_table = |line: &str| { - let trimmed = line.trim_start(); - trimmed.starts_with("[[creative_opportunities.slot]]") - || trimmed.starts_with("[creative_opportunities.slot.") - }; - let is_unrelated_table = |line: &str| { - let trimmed = line.trim_start(); - trimmed.starts_with('[') && !is_slot_table(line) && trimmed != "[creative_opportunities]" - }; - - // Where the existing slot array begins (first slot table after the header), - // else the end of the scalar block (first unrelated table, or EOF). - let existing_start = lines[header + 1..] - .iter() - .position(|line| is_slot_table(line)) - .map(|offset| header + 1 + offset); - let start = existing_start.unwrap_or_else(|| { - lines[header + 1..] - .iter() - .position(|line| is_unrelated_table(line)) - .map_or(lines.len(), |offset| header + 1 + offset) - }); - // Where the slot array ends: first unrelated top-level table, or EOF. - let end = lines[start..] - .iter() - .position(|line| is_unrelated_table(line)) - .map_or(lines.len(), |offset| start + offset); - - let mut result = lines[..start].join("\n"); - if !result.is_empty() { - result.push('\n'); - } - result.push_str(rendered); - result.push('\n'); - let tail = lines[end..].join("\n"); - if !tail.is_empty() { - result.push('\n'); - result.push_str(&tail); - } - if existing.ends_with('\n') && !result.ends_with('\n') { - result.push('\n'); - } - Ok(result) -} - -fn replace_key_in_section( - document: &str, - section: &str, - key: &str, - replacement_line: &str, -) -> CliResult { - let section_header = format!("[{section}]"); - let mut in_section = false; - let mut replaced = false; - let mut saw_section = false; - let mut lines = Vec::new(); - - for line in document.lines() { - let trimmed = line.trim(); - if trimmed.starts_with('[') && trimmed.ends_with(']') { - in_section = trimmed == section_header; - saw_section |= in_section; - } - - if in_section && !replaced && is_key_line(trimmed, key) { - lines.push(replacement_line.to_string()); - replaced = true; - } else { - lines.push(line.to_string()); - } - } - - if !saw_section { - return cli_error(format!( - "failed to update starter config because section `{section_header}` was not found" - )); - } - if !replaced { - return cli_error(format!( - "failed to update starter config because key `{key}` was not found in `{section_header}`" - )); - } - - let mut output = lines.join("\n"); - if document.ends_with('\n') { - output.push('\n'); - } - Ok(output) -} - -fn is_key_line(trimmed_line: &str, key: &str) -> bool { - trimmed_line - .strip_prefix(key) - .and_then(|remaining| remaining.trim_start().strip_prefix('=')) - .is_some() -} - #[cfg(test)] mod tests { use std::cell::Cell; @@ -1310,185 +882,30 @@ mod tests { ); } - fn discovered_header_slot() -> gpt_slots::DiscoveredSlots { + #[test] + fn render_discovered_slots_escapes_page_controlled_strings() { + // Slot fields scraped from the live page must be escaped so a quote + // cannot inject TOML into the drafted config. let registry = vec![collector::CollectedGptSlot { - gam_unit_path: "/222/homepage/header".to_string(), - div_id: "div-gpt-ad-header".to_string(), + gam_unit_path: "/222/homepage/head\"er".to_string(), + div_id: "div-gpt-ad-head\"er".to_string(), sizes: vec![(728, 90)], }]; - gpt_slots::discover_gpt_slots(®istry, &[], false) - } - - /// Rendered slot text for the discovered header slot, patterns = `/`. - fn header_rendered() -> String { - let merged = merge_slots(None, &discovered_header_slot(), &["/".to_string()], true); - render_slots(&merged) - } - - fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { - toml::from_str::(toml_str).expect("valid creative config") - } - - #[test] - fn splice_replaces_slots_and_preserves_other_sections() { - let existing = "[publisher]\ndomain = \"x\"\n\n\ - [creative_opportunities]\ngam_network_id = \"111\"\nprice_granularity = \"dense\"\n\n\ - [[creative_opportunities.slot]]\nid = \"old\"\ndiv_id = \"old\"\n\ - gam_unit_path = \"/111/old\"\npage_patterns = [\"/\"]\n\ - formats = [{ width = 300, height = 250 }]\n\n\ - [auction]\nenabled = true\n"; - - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) - .expect("should splice"); - - assert!( - out.contains("gam_network_id = \"222\""), - "network id updated" - ); - assert!(!out.contains("id = \"old\""), "old slot removed"); - assert!( - out.contains("gam_unit_path = \"/222/homepage/header\""), - "new slot written" - ); - assert!( - out.contains("[publisher]") && out.contains("domain = \"x\""), - "publisher section preserved" - ); - assert!( - out.contains("[auction]") && out.contains("enabled = true"), - "trailing auction section preserved" - ); - toml::from_str::(&out).expect("spliced config is valid TOML"); - } - - #[test] - fn splice_creates_section_when_absent() { - // Config with no [creative_opportunities] at all — generate should append it. - let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; - - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) - .expect("should splice"); - - let value = toml::from_str::(&out).expect("valid TOML"); - assert_eq!( - value["creative_opportunities"]["gam_network_id"].as_str(), - Some("222"), - "appended section carries the discovered network id" - ); - assert_eq!( - value["creative_opportunities"]["slot"][0]["id"].as_str(), - Some("header") - ); - assert!( - value["publisher"]["domain"].as_str() == Some("x") - && value["auction"]["enabled"].as_bool() == Some(true), - "existing sections preserved when appending" - ); - } - - #[test] - fn splice_inserts_when_no_existing_slots() { - let existing = - "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; + let slots = gpt_slots::discover_gpt_slots(®istry, &[], false); + let url = Url::parse("https://publisher.example/").expect("should parse URL"); - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) - .expect("should splice"); + let rendered = render_discovered_slots(&url, &slots); - let value = toml::from_str::(&out).expect("valid TOML"); + let value = toml::from_str::(&rendered) + .expect("should render valid TOML despite embedded quotes"); + let slot = &value["creative_opportunities"]["slot"][0]; assert_eq!( - value["creative_opportunities"]["slot"][0]["id"].as_str(), - Some("header"), - "inserted slot id strips the div-gpt-ad- prefix" - ); - assert_eq!( - value["creative_opportunities"]["slot"][0]["div_id"].as_str(), - Some("div-gpt-ad-header"), - "div_id keeps the stable stem" - ); - assert!( - value["auction"]["enabled"].as_bool() == Some(true), - "auction section preserved after inserted slots" + slot["div_id"].as_str(), + Some("div-gpt-ad-head\"er"), + "should keep the quote as data, not TOML syntax" ); } - #[test] - fn merge_second_run_unions_page_patterns() { - // Existing slot on "/"; re-discovered this run with "/news/*". - let existing = existing_config( - "gam_network_id = \"222\"\n\n\ - [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ - gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ - formats = [{ width = 728, height = 90 }]\n", - ); - - let merged = merge_slots( - Some(&existing), - &discovered_header_slot(), - &["/news/*".to_string()], - false, - ); - - assert_eq!(merged.len(), 1, "same slot is not duplicated"); - assert_eq!( - merged[0].page_patterns, - vec!["/".to_string(), "/news/*".to_string()], - "this run's pattern is unioned into the existing slot" - ); - } - - #[test] - fn merge_keeps_existing_only_slots() { - // Existing has header + sidebar; this run re-sees only header. - let existing = existing_config( - "gam_network_id = \"222\"\n\n\ - [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ - gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ - formats = [{ width = 728, height = 90 }]\n\n\ - [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ - gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ - formats = [{ width = 300, height = 250 }]\nfloor_price = 0.5\n", - ); - - let merged = merge_slots( - Some(&existing), - &discovered_header_slot(), - &["/".to_string()], - false, - ); - - let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); - assert_eq!(ids, vec!["header", "sidebar"], "sidebar preserved"); - let sidebar = merged - .iter() - .find(|slot| slot.id == "sidebar") - .expect("sidebar"); - assert_eq!( - sidebar.floor_price, - Some(0.5), - "hand-tuned fields preserved" - ); - } - - #[test] - fn merge_replace_wipes_existing() { - let existing = existing_config( - "gam_network_id = \"222\"\n\n\ - [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ - gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ - formats = [{ width = 300, height = 250 }]\n", - ); - - let merged = merge_slots( - Some(&existing), - &discovered_header_slot(), - &["/".to_string()], - true, - ); - - let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); - assert_eq!(ids, vec!["header"], "--replace keeps only discovered slots"); - } - #[test] fn default_page_pattern_uses_path_or_root() { assert_eq!( @@ -1500,73 +917,4 @@ mod tests { "/" ); } - - #[test] - fn resolve_network_id_prefers_discovered_unless_preserving_existing() { - let with_slots = existing_config( - "gam_network_id = \"111\"\n\n[[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\n\ - gam_unit_path = \"/111/s\"\npage_patterns = [\"/\"]\n\ - formats = [{ width = 300, height = 250 }]\n", - ); - let empty = existing_config("gam_network_id = \"111\"\n"); - - // Real merge → keep existing. - assert_eq!( - resolve_network_id(Some(&with_slots), Some("222"), false).as_deref(), - Some("111") - ); - // Placeholder section with no slots → discovered wins. - assert_eq!( - resolve_network_id(Some(&empty), Some("222"), false).as_deref(), - Some("222") - ); - // --replace → discovered wins. - assert_eq!( - resolve_network_id(Some(&with_slots), Some("222"), true).as_deref(), - Some("222") - ); - // No existing config → discovered. - assert_eq!( - resolve_network_id(None, Some("222"), false).as_deref(), - Some("222") - ); - } - - #[test] - fn toml_key_quotes_only_non_bare_keys() { - assert_eq!(toml_key("zone"), "zone"); - assert_eq!(toml_key("ad-loc"), "ad-loc"); - assert_eq!(toml_key("a.b"), "\"a.b\""); - assert_eq!(toml_key("with space"), "\"with space\""); - assert_eq!(toml_key(""), "\"\""); - } - - #[test] - fn toml_string_escapes_quotes_backslashes_and_controls() { - assert_eq!(toml_string("a\"b\\c"), "\"a\\\"b\\\\c\""); - assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); - } - - #[test] - fn render_quotes_exotic_targeting_keys_to_valid_toml() { - let existing = existing_config( - "gam_network_id = \"1\"\n\n\ - [[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\ngam_unit_path = \"/1/s\"\n\ - page_patterns = [\"/\"]\nformats = [{ width = 300, height = 250 }]\n\ - targeting = { \"a.b\" = \"x\" }\n", - ); - - let merged = merge_slots( - Some(&existing), - &discovered_header_slot(), - &["/".to_string()], - false, - ); - let doc = format!( - "[creative_opportunities]\ngam_network_id = \"1\"\n{}", - render_slots(&merged) - ); - - toml::from_str::(&doc).expect("exotic targeting key renders as valid TOML"); - } } diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs new file mode 100644 index 000000000..9c24ec9d6 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -0,0 +1,762 @@ +//! TOML-side slot config: the [`RenderSlot`] model, run merging, rendering, +//! and in-place `[creative_opportunities]` splicing for `ts audit ad-templates +//! generate`. + +use std::collections::BTreeMap; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, CreativeOpportunitySlot, +}; + +use crate::commands::audit::generate::gpt_slots; +use crate::error::{CliResult, cli_error, report_error}; + +/// A slot ready to render — the union of discovered and existing fields, without +/// the core type's `pub(crate)` compiled-pattern cache. +#[derive(Debug, Clone)] +pub(super) struct RenderSlot { + id: String, + div_id: Option, + gam_unit_path: Option, + page_patterns: Vec, + /// `(width, height, non-banner media type)`. + formats: Vec<(u32, u32, Option<&'static str>)>, + floor_price: Option, + targeting: BTreeMap, + aps_slot_id: Option, + /// `Some` when the slot runs Prebid; the map is per-bidder params (often empty). + prebid_bidders: Option>, +} + +impl RenderSlot { + /// The stable identity used to match slots across runs: the div id (or slot + /// id), with any trailing `-` trimmed so hand-authored stems still match. + fn key(&self) -> String { + self.div_id + .as_deref() + .unwrap_or(&self.id) + .trim_end_matches('-') + .to_string() + } + + fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { + Self { + id: slot.id.clone(), + div_id: Some(slot.div_id.clone()), + gam_unit_path: Some(slot.gam_unit_path.clone()), + page_patterns: patterns.to_vec(), + formats: slot + .formats + .iter() + .map(|&(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: slot.has_prebid.then(BTreeMap::new), + } + } + + fn from_existing(slot: &CreativeOpportunitySlot) -> Self { + Self { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + gam_unit_path: slot.gam_unit_path.clone(), + page_patterns: slot.page_patterns.clone(), + formats: slot + .formats + .iter() + .map(|format| { + ( + format.width, + format.height, + media_type_label(&format.media_type), + ) + }) + .collect(), + floor_price: slot.floor_price, + targeting: slot + .targeting + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), + prebid_bidders: slot.providers.prebid.as_ref().map(|prebid| { + prebid + .bidders + .iter() + .map(|(name, params)| (name.clone(), params.clone())) + .collect() + }), + } + } +} + +/// The non-default (non-banner) media-type label to emit, or `None` for banner. +fn media_type_label(media_type: &MediaType) -> Option<&'static str> { + match media_type { + MediaType::Banner => None, + MediaType::Video => Some("video"), + MediaType::Native => Some("native"), + } +} + +/// Merges discovered slots into the existing slot set, keyed by [`RenderSlot::key`]. +/// +/// - `--replace` (or no existing slots): the result is exactly the discovered set. +/// - Otherwise existing slots are preserved (covering other pages / hand-tuned +/// fields); a slot re-seen this run has `run_patterns` unioned into its +/// `page_patterns`; slots seen only this run are appended. +pub(super) fn merge_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered: &gpt_slots::DiscoveredSlots, + run_patterns: &[String], + replace: bool, +) -> Vec { + let discovered_slots: Vec = discovered + .slots + .iter() + .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) + .collect(); + + let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); + if replace || existing_slots.is_empty() { + return discovered_slots; + } + + let mut merged: Vec = existing_slots + .iter() + .map(RenderSlot::from_existing) + .collect(); + for slot in discovered_slots { + let key = slot.key(); + if let Some(present) = merged.iter_mut().find(|existing| existing.key() == key) { + for pattern in &slot.page_patterns { + if !present.page_patterns.contains(pattern) { + present.page_patterns.push(pattern.clone()); + } + } + } else { + merged.push(slot); + } + } + merged +} + +/// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. +pub(super) fn render_slots(slots: &[RenderSlot]) -> String { + let mut out = String::from( + "\n# Slots managed by `ts audit ad-templates generate`.\n\ + # Review page_patterns and formats before validating/pushing.\n", + ); + for slot in slots { + out.push_str("\n[[creative_opportunities.slot]]\n"); + out.push_str(&format!("id = {}\n", toml_string(&slot.id))); + if let Some(div_id) = &slot.div_id { + out.push_str(&format!("div_id = {}\n", toml_string(div_id))); + } + if let Some(path) = &slot.gam_unit_path { + out.push_str(&format!("gam_unit_path = {}\n", toml_string(path))); + } + let patterns = slot + .page_patterns + .iter() + .map(|pattern| toml_string(pattern)) + .collect::>() + .join(", "); + out.push_str(&format!("page_patterns = [{patterns}]\n")); + let formats = slot + .formats + .iter() + .map(|(width, height, media_type)| match media_type { + Some(kind) => { + format!("{{ width = {width}, height = {height}, media_type = \"{kind}\" }}") + } + None => format!("{{ width = {width}, height = {height} }}"), + }) + .collect::>() + .join(", "); + out.push_str(&format!("formats = [{formats}]\n")); + if let Some(floor) = slot.floor_price { + // `f64` Display prints `NaN`, which is not valid TOML (`nan` is); + // normalize non-finite values so the spliced config stays parseable. + if floor.is_finite() { + out.push_str(&format!("floor_price = {floor}\n")); + } else if floor.is_nan() { + out.push_str("floor_price = nan\n"); + } else if floor.is_sign_positive() { + out.push_str("floor_price = inf\n"); + } else { + out.push_str("floor_price = -inf\n"); + } + } + if !slot.targeting.is_empty() { + let pairs = slot + .targeting + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_string(value))) + .collect::>() + .join(", "); + out.push_str(&format!("targeting = {{ {pairs} }}\n")); + } + if let Some(slot_id) = &slot.aps_slot_id { + out.push_str("[creative_opportunities.slot.providers.aps]\n"); + out.push_str(&format!("slot_id = {}\n", toml_string(slot_id))); + } + if let Some(bidders) = &slot.prebid_bidders { + out.push_str("[creative_opportunities.slot.providers.prebid]\n"); + let rendered = bidders + .iter() + .map(|(name, params)| format!("{} = {}", toml_key(name), toml_inline_value(params))) + .collect::>() + .join(", "); + if rendered.is_empty() { + out.push_str("bidders = {}\n"); + } else { + out.push_str(&format!("bidders = {{ {rendered} }}\n")); + } + } + } + out +} + +/// Quotes and escapes a string as a TOML basic string, including control chars. +pub(super) fn toml_string(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + control if (control as u32) < 0x20 => { + out.push_str(&format!("\\u{:04X}", control as u32)); + } + other => out.push(other), + } + } + out.push('"'); + out +} + +/// Renders a TOML table key: bare when it is a valid bare key, else a quoted key. +fn toml_key(key: &str) -> String { + let is_bare = !key.is_empty() + && key + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'); + if is_bare { + key.to_string() + } else { + toml_string(key) + } +} + +/// Renders a JSON value as a compact inline TOML value (for prebid bidder params). +fn toml_inline_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Null => "{}".to_string(), + serde_json::Value::Bool(bool) => bool.to_string(), + serde_json::Value::Number(number) => number.to_string(), + serde_json::Value::String(string) => toml_string(string), + serde_json::Value::Array(items) => { + let rendered = items + .iter() + .map(toml_inline_value) + .collect::>() + .join(", "); + format!("[{rendered}]") + } + serde_json::Value::Object(map) => { + let rendered = map + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_inline_value(value))) + .collect::>() + .join(", "); + format!("{{ {rendered} }}") + } + } +} + +/// Rewrites the `[creative_opportunities]` slot array of `existing` with the +/// pre-rendered `rendered_slots` text, updating `gam_network_id` and preserving +/// all other sections and comments. +/// +/// If the config has no `[creative_opportunities]` section, a fresh one is +/// appended so `generate` works against a config that omits it. +pub(super) fn splice_creative_slots( + existing: &str, + network_id: Option<&str>, + rendered_slots: &str, +) -> CliResult { + let rendered = rendered_slots.trim_matches('\n'); + + // No section yet — append a fresh one with the network id and slots. + if !existing + .lines() + .any(|line| line.trim() == "[creative_opportunities]") + { + let mut result = existing.to_string(); + if !result.is_empty() && !result.ends_with('\n') { + result.push('\n'); + } + result.push_str("\n[creative_opportunities]\n"); + if let Some(network_id) = network_id { + result.push_str(&format!("gam_network_id = {}\n", toml_string(network_id))); + } + result.push_str(rendered); + result.push('\n'); + return Ok(result); + } + + // Section exists — update `gam_network_id` (best-effort) and replace slots. + let mut document = existing.to_string(); + if let Some(network_id) = network_id + && let Ok(updated) = replace_key_in_section( + &document, + "creative_opportunities", + "gam_network_id", + &format!("gam_network_id = {}", toml_string(network_id)), + ) + { + document = updated; + } + + let lines: Vec<&str> = document.lines().collect(); + let header = lines + .iter() + .position(|line| line.trim() == "[creative_opportunities]") + .ok_or_else(|| { + report_error("target config has no [creative_opportunities] section to update") + })?; + + let is_slot_table = |line: &str| { + let trimmed = line.trim_start(); + trimmed.starts_with("[[creative_opportunities.slot]]") + || trimmed.starts_with("[creative_opportunities.slot.") + }; + let is_unrelated_table = |line: &str| { + let trimmed = line.trim_start(); + trimmed.starts_with('[') && !is_slot_table(line) && trimmed != "[creative_opportunities]" + }; + + // Where the existing slot array begins (first slot table after the header), + // else the end of the scalar block (first unrelated table, or EOF). + let existing_start = lines[header + 1..] + .iter() + .position(|line| is_slot_table(line)) + .map(|offset| header + 1 + offset); + let start = existing_start.unwrap_or_else(|| { + lines[header + 1..] + .iter() + .position(|line| is_unrelated_table(line)) + .map_or(lines.len(), |offset| header + 1 + offset) + }); + // Where the slot array ends: first unrelated top-level table, or EOF. + let end = lines[start..] + .iter() + .position(|line| is_unrelated_table(line)) + .map_or(lines.len(), |offset| start + offset); + + let mut result = lines[..start].join("\n"); + if !result.is_empty() { + result.push('\n'); + } + result.push_str(rendered); + result.push('\n'); + let tail = lines[end..].join("\n"); + if !tail.is_empty() { + result.push('\n'); + result.push_str(&tail); + } + if existing.ends_with('\n') && !result.ends_with('\n') { + result.push('\n'); + } + if uses_crlf(existing) { + result = result.replace('\n', "\r\n"); + } + Ok(result) +} + +/// Whether `document` uses CRLF line endings (so edits preserve them). +fn uses_crlf(document: &str) -> bool { + document.contains("\r\n") +} + +pub(super) fn replace_key_in_section( + document: &str, + section: &str, + key: &str, + replacement_line: &str, +) -> CliResult { + let section_header = format!("[{section}]"); + let mut in_section = false; + let mut replaced = false; + let mut saw_section = false; + let mut lines = Vec::new(); + + for line in document.lines() { + let trimmed = line.trim(); + if trimmed.starts_with('[') && trimmed.ends_with(']') { + in_section = trimmed == section_header; + saw_section |= in_section; + } + + if in_section && !replaced && is_key_line(trimmed, key) { + lines.push(replacement_line.to_string()); + replaced = true; + } else { + lines.push(line.to_string()); + } + } + + if !saw_section { + return cli_error(format!( + "failed to update starter config because section `{section_header}` was not found" + )); + } + if !replaced { + return cli_error(format!( + "failed to update starter config because key `{key}` was not found in `{section_header}`" + )); + } + + let mut output = lines.join("\n"); + if document.ends_with('\n') { + output.push('\n'); + } + if uses_crlf(document) { + // `lines()` stripped the `\r`s; restore the document's CRLF endings. + output = output.replace("\r\n", "\n").replace('\n', "\r\n"); + } + Ok(output) +} + +fn is_key_line(trimmed_line: &str, key: &str) -> bool { + trimmed_line + .strip_prefix(key) + .and_then(|remaining| remaining.trim_start().strip_prefix('=')) + .is_some() +} + +/// Chooses the `gam_network_id` to write. +/// +/// The existing id is kept only when a real merge preserves existing slots. +/// On `--replace`, or when the config had no slots (e.g. a placeholder +/// `[creative_opportunities]` section), the discovered id wins — mirroring +/// [`merge_slots`], which returns discovered-only in those cases. +pub(super) fn resolve_network_id( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_network_id: Option<&str>, + replace: bool, +) -> Option { + let existing_network_id = existing.map(|config| config.gam_network_id.clone()); + let preserving_existing = !replace && existing.is_some_and(|config| !config.slot.is_empty()); + if preserving_existing { + existing_network_id.or_else(|| discovered_network_id.map(str::to_string)) + } else { + discovered_network_id + .map(str::to_string) + .or(existing_network_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector; + + fn discovered_header_slot() -> gpt_slots::DiscoveredSlots { + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + gpt_slots::discover_gpt_slots(®istry, &[], false) + } + + /// Rendered slot text for the discovered header slot, patterns = `/`. + fn header_rendered() -> String { + let merged = merge_slots(None, &discovered_header_slot(), &["/".to_string()], true); + render_slots(&merged) + } + + fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { + toml::from_str::(toml_str).expect("valid creative config") + } + + #[test] + fn splice_replaces_slots_and_preserves_other_sections() { + let existing = "[publisher]\ndomain = \"x\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\nprice_granularity = \"dense\"\n\n\ + [[creative_opportunities.slot]]\nid = \"old\"\ndiv_id = \"old\"\n\ + gam_unit_path = \"/111/old\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + assert!( + out.contains("gam_network_id = \"222\""), + "network id updated" + ); + assert!(!out.contains("id = \"old\""), "old slot removed"); + assert!( + out.contains("gam_unit_path = \"/222/homepage/header\""), + "new slot written" + ); + assert!( + out.contains("[publisher]") && out.contains("domain = \"x\""), + "publisher section preserved" + ); + assert!( + out.contains("[auction]") && out.contains("enabled = true"), + "trailing auction section preserved" + ); + toml::from_str::(&out).expect("spliced config is valid TOML"); + } + + #[test] + fn splice_preserves_crlf_line_endings() { + let existing = "[creative_opportunities]\r\ngam_network_id = \"111\"\r\n\r\n\ + [auction]\r\nenabled = true\r\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "every line ending should stay CRLF" + ); + let value = toml::from_str::(&out).expect("spliced CRLF config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated in CRLF config" + ); + } + + #[test] + fn render_slots_writes_non_finite_floor_price_as_valid_toml() { + let slot = RenderSlot { + id: "header".to_string(), + div_id: Some("div-gpt-ad-header".to_string()), + gam_unit_path: Some("/222/homepage/header".to_string()), + page_patterns: vec!["/".to_string()], + formats: vec![(728, 90, None)], + floor_price: Some(f64::NAN), + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: None, + }; + + let rendered = render_slots(&[slot]); + + assert!( + rendered.contains("floor_price = nan"), + "NaN should render as TOML `nan`, not Rust `NaN`" + ); + toml::from_str::(&rendered).expect("rendered slots are valid TOML"); + } + + #[test] + fn splice_creates_section_when_absent() { + // Config with no [creative_opportunities] at all — generate should append it. + let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "appended section carries the discovered network id" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert!( + value["publisher"]["domain"].as_str() == Some("x") + && value["auction"]["enabled"].as_bool() == Some(true), + "existing sections preserved when appending" + ); + } + + #[test] + fn splice_inserts_when_no_existing_slots() { + let existing = + "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header"), + "inserted slot id strips the div-gpt-ad- prefix" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "div_id keeps the stable stem" + ); + assert!( + value["auction"]["enabled"].as_bool() == Some(true), + "auction section preserved after inserted slots" + ); + } + + #[test] + fn merge_second_run_unions_page_patterns() { + // Existing slot on "/"; re-discovered this run with "/news/*". + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/news/*".to_string()], + false, + ); + + assert_eq!(merged.len(), 1, "same slot is not duplicated"); + assert_eq!( + merged[0].page_patterns, + vec!["/".to_string(), "/news/*".to_string()], + "this run's pattern is unioned into the existing slot" + ); + } + + #[test] + fn merge_keeps_existing_only_slots() { + // Existing has header + sidebar; this run re-sees only header. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 0.5\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header", "sidebar"], "sidebar preserved"); + let sidebar = merged + .iter() + .find(|slot| slot.id == "sidebar") + .expect("sidebar"); + assert_eq!( + sidebar.floor_price, + Some(0.5), + "hand-tuned fields preserved" + ); + } + + #[test] + fn merge_replace_wipes_existing() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + true, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header"], "--replace keeps only discovered slots"); + } + + #[test] + fn resolve_network_id_prefers_discovered_unless_preserving_existing() { + let with_slots = existing_config( + "gam_network_id = \"111\"\n\n[[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\n\ + gam_unit_path = \"/111/s\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let empty = existing_config("gam_network_id = \"111\"\n"); + + // Real merge → keep existing. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), false).as_deref(), + Some("111") + ); + // Placeholder section with no slots → discovered wins. + assert_eq!( + resolve_network_id(Some(&empty), Some("222"), false).as_deref(), + Some("222") + ); + // --replace → discovered wins. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), true).as_deref(), + Some("222") + ); + // No existing config → discovered. + assert_eq!( + resolve_network_id(None, Some("222"), false).as_deref(), + Some("222") + ); + } + + #[test] + fn toml_key_quotes_only_non_bare_keys() { + assert_eq!(toml_key("zone"), "zone"); + assert_eq!(toml_key("ad-loc"), "ad-loc"); + assert_eq!(toml_key("a.b"), "\"a.b\""); + assert_eq!(toml_key("with space"), "\"with space\""); + assert_eq!(toml_key(""), "\"\""); + } + + #[test] + fn toml_string_escapes_quotes_backslashes_and_controls() { + assert_eq!(toml_string("a\"b\\c"), "\"a\\\"b\\\\c\""); + assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); + } + + #[test] + fn render_quotes_exotic_targeting_keys_to_valid_toml() { + let existing = existing_config( + "gam_network_id = \"1\"\n\n\ + [[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\ngam_unit_path = \"/1/s\"\n\ + page_patterns = [\"/\"]\nformats = [{ width = 300, height = 250 }]\n\ + targeting = { \"a.b\" = \"x\" }\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + let doc = format!( + "[creative_opportunities]\ngam_network_id = \"1\"\n{}", + render_slots(&merged) + ); + + toml::from_str::(&doc).expect("exotic targeting key renders as valid TOML"); + } +} diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 047261153..a85ab9a44 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -618,7 +618,7 @@ mod tests { // input combination, `expected == Yes` must equal the legacy all-AND boolean. #[test] fn ad_stack_gate_with_known_consent_matches_legacy_boolean() { - for bits in 0u8..64 { + for bits in 0u8..128 { let input = AdStackGateInput { method_get: bits & 1 != 0, navigation: bits & 2 != 0, @@ -626,7 +626,7 @@ mod tests { bot: bits & 8 != 0, matched_slots: bits & 16 != 0, consent_allows_auction: Some(bits & 32 != 0), - auction_enabled: bits & 1 == 0, + auction_enabled: bits & 64 != 0, }; // Legacy semantics: all positive gates true, both negative gates false. let legacy = input.method_get From db302cbae4aec2ab544875a0c8aed9f2ea8e78db Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 8 Jul 2026 11:23:47 +0530 Subject: [PATCH 137/395] Address ad-template generate review findings - Recognize [creative_opportunities] table headers carrying inline comments in the in-place splice and replace_key_in_section, so a valid operator config is updated instead of gaining a duplicate section - Default generated page_patterns from the recorded post-redirect final URL instead of the requested URL, falling back to the requested URL when the recorded final URL is invalid - Escape DEL (U+007F) in toml_string, which TOML basic strings reject alongside chars below U+0020 Each fix carries a parse-backed regression test. --- .../src/commands/audit/generate/mod.rs | 51 ++++++++- .../src/commands/audit/generate/slot_toml.rs | 104 +++++++++++++++++- 2 files changed, 147 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index a6f2546e6..5f58104cc 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -478,9 +478,13 @@ pub(crate) fn run_update_slots( } // Patterns for slots seen on this run: the `--page-pattern` values, or the - // audited path when none are given (preserving single-page behavior). + // audited path when none are given (preserving single-page behavior). The + // default uses the recorded post-redirect URL so it matches the page that + // was actually audited, falling back to the requested URL when the + // recorded final URL is invalid. let run_patterns: Vec = if page_patterns.is_empty() { - vec![default_page_pattern(&target_url)] + let audited_url = collected.final_url().unwrap_or_else(|_| target_url.clone()); + vec![default_page_pattern(&audited_url)] } else { page_patterns.to_vec() }; @@ -906,6 +910,49 @@ mod tests { ); } + #[test] + fn update_slots_defaults_pattern_to_final_url_after_redirect() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + // The requested URL redirects; slots are scraped from the final page. + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/news/story".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + "https://publisher.example/", + &config_path, + None, + &[], + false, + &[], + false, + &collector, + &mut out, + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["page_patterns"][0].as_str(), + Some("/news/story"), + "default pattern should use the post-redirect path, not the requested one" + ); + } + #[test] fn default_page_pattern_uses_path_or_root() { assert_eq!( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 9c24ec9d6..0fc85fcc1 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -232,7 +232,8 @@ pub(super) fn toml_string(value: &str) -> String { '\n' => out.push_str("\\n"), '\r' => out.push_str("\\r"), '\t' => out.push_str("\\t"), - control if (control as u32) < 0x20 => { + // TOML basic strings reject U+0000..U+001F and DEL (U+007F). + control if (control as u32) < 0x20 || control == '\u{7f}' => { out.push_str(&format!("\\u{:04X}", control as u32)); } other => out.push(other), @@ -297,7 +298,7 @@ pub(super) fn splice_creative_slots( // No section yet — append a fresh one with the network id and slots. if !existing .lines() - .any(|line| line.trim() == "[creative_opportunities]") + .any(|line| is_table_header(line, "[creative_opportunities]")) { let mut result = existing.to_string(); if !result.is_empty() && !result.ends_with('\n') { @@ -328,7 +329,7 @@ pub(super) fn splice_creative_slots( let lines: Vec<&str> = document.lines().collect(); let header = lines .iter() - .position(|line| line.trim() == "[creative_opportunities]") + .position(|line| is_table_header(line, "[creative_opportunities]")) .ok_or_else(|| { report_error("target config has no [creative_opportunities] section to update") })?; @@ -340,7 +341,9 @@ pub(super) fn splice_creative_slots( }; let is_unrelated_table = |line: &str| { let trimmed = line.trim_start(); - trimmed.starts_with('[') && !is_slot_table(line) && trimmed != "[creative_opportunities]" + trimmed.starts_with('[') + && !is_slot_table(line) + && !is_table_header(line, "[creative_opportunities]") }; // Where the existing slot array begins (first slot table after the header), @@ -386,6 +389,25 @@ fn uses_crlf(document: &str) -> bool { document.contains("\r\n") } +/// Strips a trailing inline `# comment` from a candidate table-header line. +/// +/// Only valid on header candidates: header lines cannot contain `#` before the +/// closing bracket unless it is inside a quoted key, which the configs this +/// updater manages never use. +fn strip_inline_comment(line: &str) -> &str { + match line.find('#') { + Some(position) => line[..position].trim_end(), + None => line, + } +} + +/// Whether `line` is exactly the `section_header` table header (for example +/// `[creative_opportunities]`), tolerating surrounding whitespace and a +/// trailing inline `# comment` — both valid TOML. +fn is_table_header(line: &str, section_header: &str) -> bool { + strip_inline_comment(line.trim()) == section_header +} + pub(super) fn replace_key_in_section( document: &str, section: &str, @@ -400,8 +422,9 @@ pub(super) fn replace_key_in_section( for line in document.lines() { let trimmed = line.trim(); - if trimmed.starts_with('[') && trimmed.ends_with(']') { - in_section = trimmed == section_header; + let header_candidate = strip_inline_comment(trimmed); + if header_candidate.starts_with('[') && header_candidate.ends_with(']') { + in_section = header_candidate == section_header; saw_section |= in_section; } @@ -588,6 +611,40 @@ mod tests { ); } + #[test] + fn splice_recognizes_inline_commented_section_header() { + // `[creative_opportunities] # comment` is valid TOML; the splice must + // update it in place instead of appending a duplicate section. + let existing = "[creative_opportunities] # ad templates\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + assert_eq!( + out.lines() + .filter(|line| is_table_header(line, "[creative_opportunities]")) + .count(), + 1, + "commented header must not be duplicated" + ); + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated under a commented header" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "commented trailing section preserved" + ); + } + #[test] fn splice_inserts_when_no_existing_slots() { let existing = @@ -737,6 +794,41 @@ mod tests { assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); } + #[test] + fn toml_string_escapes_del_control_char() { + assert_eq!(toml_string("a\u{7f}b"), "\"a\\u007Fb\""); + let doc = format!("value = {}", toml_string("a\u{7f}b")); + let value = toml::from_str::(&doc).expect("DEL escapes to valid TOML"); + assert_eq!( + value["value"].as_str(), + Some("a\u{7f}b"), + "escaped DEL round-trips as data" + ); + } + + #[test] + fn replace_key_handles_inline_commented_headers() { + let document = "[creative_opportunities] # managed\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let updated = replace_key_in_section( + document, + "creative_opportunities", + "gam_network_id", + "gam_network_id = \"222\"", + ) + .expect("should find the commented section header"); + + assert!( + updated.contains("gam_network_id = \"222\""), + "key replaced under a commented header" + ); + assert!( + updated.contains("enabled = true"), + "later commented section left untouched" + ); + } + #[test] fn render_quotes_exotic_targeting_keys_to_valid_toml() { let existing = existing_config( From ccccfa7b7d7aa91812524d5a516e9cdde73375de Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 9 Jul 2026 16:04:42 +0530 Subject: [PATCH 138/395] Resolve ad-template CLI review findings --- Cargo.lock | 1 + README.md | 2 +- crates/trusted-server-cli/Cargo.toml | 1 + crates/trusted-server-cli/src/app_config.rs | 23 +- .../src/commands/audit/generate/gpt_slots.rs | 113 +++++++++- .../src/commands/audit/generate/mod.rs | 95 ++++++++ .../src/commands/audit/generate/slot_toml.rs | 205 +++++++++++++++++- .../src/commands/audit/mod.rs | 98 ++++++++- .../src/commands/audit/page.rs | 10 - crates/trusted-server-cli/src/run.rs | 32 ++- docs/guide/cli.md | 11 +- docs/guide/getting-started.md | 2 +- ...6-26-server-side-ad-template-cli-design.md | 16 +- 13 files changed, 563 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6036b24f8..a20139cb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5261,6 +5261,7 @@ dependencies = [ "scraper", "serde", "serde_json", + "temp-env", "tempfile", "time", "tokio", diff --git a/README.md b/README.md index e56937e89..405822061 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ ts config init ts config validate # Audit a public page with Chrome/Chromium to bootstrap a draft config -ts audit https://publisher.example +ts audit generate https://publisher.example # Run tests (Fastly/WASM crates — requires Viceroy) cargo test-fastly diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index 276b4fd95..4a643d950 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -61,4 +61,5 @@ webpki-roots = { workspace = true } x509-parser = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +temp-env = { workspace = true } tempfile = { workspace = true } diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs index b84d16747..fadf58cde 100644 --- a/crates/trusted-server-cli/src/app_config.rs +++ b/crates/trusted-server-cli/src/app_config.rs @@ -49,6 +49,27 @@ pub struct LoadedSettings { /// explicit `--app-config` path is given and is missing, the error names that /// exact path rather than silently falling back. pub fn load_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, !args.no_env) +} + +/// Loads Trusted Server settings from the resolved app-config file without +/// applying environment overlays. +/// +/// Mutating commands use this path so environment-only values are never +/// persisted into the operator-owned TOML file. +/// +/// # Errors +/// +/// Returns the same path-resolution, read, and parse errors as +/// [`load_settings`]. +pub fn load_file_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, false) +} + +fn load_settings_with_env_overlay( + args: &AppConfigArgs, + env_overlay: bool, +) -> Result { let manifest_loader = ManifestLoader::from_path(&args.manifest) .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { @@ -61,7 +82,7 @@ pub fn load_settings(args: &AppConfigArgs) -> Result { resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); let mut opts = AppConfigLoadOptions::default(); - opts.env_overlay = !args.no_env; + opts.env_overlay = env_overlay; let app_config = app_config::deserialize_app_config_with_options::( &app_config_path, &app_name, diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index b7d595dbc..fea34dd1f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -20,6 +20,7 @@ use std::collections::BTreeSet; use std::sync::LazyLock; use regex::Regex; +use trusted_server_core::creative_opportunities::validate_slot_id; use url::Url; use crate::commands::audit::generate::collector::{CollectedGptSlot, CollectedRequest}; @@ -113,6 +114,7 @@ pub(crate) fn discover_gpt_slots( } slots.push(slot); } + make_slot_ids_unique(&mut slots); DiscoveredSlots { gam_network_id, @@ -274,12 +276,56 @@ fn parse_sizes(raw: &str) -> Vec<(u32, u32)> { sizes } -/// Derives a slot id from a div id by stripping the common GPT prefix. +/// Derives a runtime-safe slot id from a div id. +/// +/// The common GPT prefix is stripped, invalid character runs become one +/// hyphen, and an all-invalid value falls back to `slot`. fn slot_id_from_div(div_id: &str) -> String { - div_id - .strip_prefix(GPT_DIV_PREFIX) - .unwrap_or(div_id) - .to_string() + let candidate = div_id.strip_prefix(GPT_DIV_PREFIX).unwrap_or(div_id); + let mut id = String::with_capacity(candidate.len()); + let mut previous_was_hyphen = false; + for character in candidate.chars() { + if character.is_ascii_alphanumeric() || character == '_' { + id.push(character); + previous_was_hyphen = false; + } else if !id.is_empty() && !previous_was_hyphen { + id.push('-'); + previous_was_hyphen = true; + } + } + while id.ends_with('-') { + id.pop(); + } + if id.is_empty() { + id.push_str("slot"); + } + + if validate_slot_id(&id).is_ok() { + id + } else { + "slot".to_string() + } +} + +/// Adds deterministic numeric suffixes when sanitization produces duplicate ids. +fn make_slot_ids_unique(slots: &mut [DiscoveredSlot]) { + let mut used = BTreeSet::new(); + for slot in slots { + if used.insert(slot.id.clone()) { + continue; + } + + let base = slot.id.clone(); + let mut suffix = 2_usize; + loop { + let candidate = format!("{base}-{suffix}"); + if used.insert(candidate.clone()) { + slot.id = candidate; + break; + } + suffix += 1; + } + } } /// Detects Prebid/header-bidding signals in a slot's `prev_scp` targeting. @@ -549,6 +595,63 @@ mod tests { ); } + #[test] + fn sanitizes_page_controlled_div_ids_for_runtime_slot_ids() { + let registry = vec![registry_slot( + "/123456789/homepage/header", + "div-gpt-ad-header.main: 1", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "header-main-1"); + assert_eq!( + discovered.slots[0].div_id, "div-gpt-ad-header.main: 1", + "matching should retain the original normalized div stem" + ); + trusted_server_core::creative_opportunities::validate_slot_id(&discovered.slots[0].id) + .expect("generated id should pass runtime validation"); + } + + #[test] + fn uses_fallback_for_div_id_without_safe_slot_id_characters() { + let registry = vec![registry_slot( + "/123456789/homepage/fallback", + "div-gpt-ad-...", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "slot"); + } + + #[test] + fn makes_colliding_sanitized_slot_ids_unique() { + let registry = vec![ + registry_slot( + "/123456789/homepage/dotted", + "div-gpt-ad-header.main", + &[(728, 90)], + ), + registry_slot( + "/123456789/homepage/colon", + "div-gpt-ad-header:main", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + let ids = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + #[test] fn normalizes_react_and_hex_hashes_to_stable_prefixes() { assert_eq!( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 5f58104cc..43a8e7f8f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -535,9 +535,11 @@ mod tests { use tempfile::TempDir; use super::*; + use crate::app_config::AppConfigArgs; use crate::commands::audit::generate::collector::{ CollectedPage, CollectedRequest, CollectedScriptTag, }; + use crate::commands::config::init::EXAMPLE_CONFIG; struct FakeCollector { collected: CollectedPage, @@ -953,6 +955,99 @@ mod tests { ); } + #[test] + fn update_slots_dry_run_does_not_persist_environment_overlay_config() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let config = EXAMPLE_CONFIG + .replace( + "replace-with-admin-password-32-bytes", + "test-admin-password-32-bytes-minimum", + ) + .replace( + "trusted-server-placeholder-secret", + "test-ec-passphrase-32-bytes-minimum", + ) + .replace( + "change-me-proxy-secret", + "test-proxy-secret-32-bytes-minimum", + ); + let config = format!( + "{config}\n\ + [[creative_opportunities.slot]]\n\ + id = \"file-only\"\n\ + div_id = \"div-gpt-ad-file\"\n\ + gam_unit_path = \"/123456789/homepage/file\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 728, height = 90 }}]\n" + ); + fs::write(&config_path, config).expect("should write config"); + let args = AppConfigArgs { + app_config: Some(config_path.clone()), + manifest: manifest_path, + no_env: false, + }; + + temp_env::with_var( + "TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__GAM_NETWORK_ID", + Some("987654321"), + || { + let effective = crate::app_config::load_settings(&args) + .expect("should load effective settings"); + assert_eq!( + effective + .settings + .creative_opportunities + .as_ref() + .expect("should have creative config") + .gam_network_id, + "987654321", + "test environment should override the network id" + ); + let loaded = crate::app_config::load_file_settings(&args) + .expect("should load file-only settings"); + let mut collected = collected_page(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/123456789/homepage/file".to_string(), + div_id: "div-gpt-ad-file".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + "https://publisher.example/", + &loaded.app_config_path, + loaded.settings.creative_opportunities.as_ref(), + &[], + false, + &[], + true, + &collector, + &mut out, + ) + .expect("should render dry-run update"); + + let output = String::from_utf8(out).expect("output should be UTF-8"); + assert!( + output.contains("id = \"file-only\""), + "dry run should preserve the file-backed slot" + ); + assert!( + output.contains("gam_network_id = \"123456789\""), + "dry run should preserve the file-backed network id" + ); + assert!( + !output.contains("987654321"), + "dry run must not persist environment-only config" + ); + }, + ); + } + #[test] fn default_page_pattern_uses_path_or_root() { assert_eq!( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 0fc85fcc1..b15e6ed9f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -4,6 +4,7 @@ use std::collections::BTreeMap; +use toml_edit::{DocumentMut, Item}; use trusted_server_core::auction::types::MediaType; use trusted_server_core::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunitySlot, @@ -30,8 +31,8 @@ pub(super) struct RenderSlot { } impl RenderSlot { - /// The stable identity used to match slots across runs: the div id (or slot - /// id), with any trailing `-` trimmed so hand-authored stems still match. + /// The stable exact identity fallback used when no configured div prefix + /// matches a discovered slot. fn key(&self) -> String { self.div_id .as_deref() @@ -129,21 +130,64 @@ pub(super) fn merge_slots( .iter() .map(RenderSlot::from_existing) .collect(); - for slot in discovered_slots { - let key = slot.key(); - if let Some(present) = merged.iter_mut().find(|existing| existing.key() == key) { + for mut slot in discovered_slots { + if let Some(index) = matching_slot_index(&merged, &slot) { + let present = &mut merged[index]; for pattern in &slot.page_patterns { if !present.page_patterns.contains(pattern) { present.page_patterns.push(pattern.clone()); } } } else { + slot.id = unique_slot_id(&slot.id, &merged); merged.push(slot); } } merged } +fn unique_slot_id(candidate: &str, existing: &[RenderSlot]) -> String { + if existing.iter().all(|slot| slot.id != candidate) { + return candidate.to_string(); + } + + let mut suffix = 2_usize; + loop { + let unique = format!("{candidate}-{suffix}"); + if existing.iter().all(|slot| slot.id != unique) { + return unique; + } + suffix += 1; + } +} + +/// Finds the most specific configured slot matching a discovered live div. +/// +/// Configured `div_id` values are runtime prefixes. Exact matches naturally +/// win because they are the longest possible prefix; equal-length ties retain +/// config order. The prior exact key behavior remains as a fallback. +fn matching_slot_index(existing: &[RenderSlot], discovered: &RenderSlot) -> Option { + if let Some(discovered_div) = discovered.div_id.as_deref() { + let mut best = None; + let mut best_length = 0; + for (index, slot) in existing.iter().enumerate() { + let Some(prefix) = slot.div_id.as_deref().filter(|prefix| !prefix.is_empty()) else { + continue; + }; + if discovered_div.starts_with(prefix) && prefix.len() > best_length { + best = Some(index); + best_length = prefix.len(); + } + } + if best.is_some() { + return best; + } + } + + let key = discovered.key(); + existing.iter().position(|slot| slot.key() == key) +} + /// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. pub(super) fn render_slots(slots: &[RenderSlot]) -> String { let mut out = String::from( @@ -294,13 +338,14 @@ pub(super) fn splice_creative_slots( rendered_slots: &str, ) -> CliResult { let rendered = rendered_slots.trim_matches('\n'); + let existing = remove_inline_slot_value(existing)?; // No section yet — append a fresh one with the network id and slots. if !existing .lines() .any(|line| is_table_header(line, "[creative_opportunities]")) { - let mut result = existing.to_string(); + let mut result = existing; if !result.is_empty() && !result.ends_with('\n') { result.push('\n'); } @@ -314,7 +359,7 @@ pub(super) fn splice_creative_slots( } // Section exists — update `gam_network_id` (best-effort) and replace slots. - let mut document = existing.to_string(); + let mut document = existing.clone(); if let Some(network_id) = network_id && let Ok(updated) = replace_key_in_section( &document, @@ -378,12 +423,37 @@ pub(super) fn splice_creative_slots( if existing.ends_with('\n') && !result.ends_with('\n') { result.push('\n'); } - if uses_crlf(existing) { + if uses_crlf(&existing) { result = result.replace('\n', "\r\n"); } Ok(result) } +/// Removes a scalar `creative_opportunities.slot` value so it can be replaced +/// with the generated array-of-tables representation. +fn remove_inline_slot_value(document: &str) -> CliResult { + let mut parsed = document.parse::().map_err(|error| { + report_error(format!( + "failed to parse target config before updating slots: {error}" + )) + })?; + let Some(creative) = parsed.get_mut("creative_opportunities") else { + return Ok(document.to_string()); + }; + let Some(table) = creative.as_table_like_mut() else { + return Ok(document.to_string()); + }; + let has_inline_slot = table + .get("slot") + .is_some_and(|slot| matches!(slot, Item::Value(_))); + if !has_inline_slot { + return Ok(document.to_string()); + } + + table.remove("slot"); + Ok(parsed.to_string()) +} + /// Whether `document` uses CRLF line endings (so edits preserve them). fn uses_crlf(document: &str) -> bool { document.contains("\r\n") @@ -670,6 +740,46 @@ mod tests { ); } + #[test] + fn splice_replaces_inline_slot_array() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = [{ id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should replace inline slot array"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "unrelated tables should be preserved" + ); + } + + #[test] + fn splice_replaces_inline_slot_map() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = { \"0\" = { id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] } }\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should replace inline slot map"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + } + #[test] fn merge_second_run_unions_page_patterns() { // Existing slot on "/"; re-discovered this run with "/news/*". @@ -695,6 +805,85 @@ mod tests { ); } + #[test] + fn merge_uses_longest_existing_div_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/broad/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"atf\"\ndiv_id = \"ad-atf-\"\n\ + gam_unit_path = \"/222/atf\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + + assert_eq!( + merged.len(), + 2, + "prefix match should not append a duplicate" + ); + let broad = merged + .iter() + .find(|slot| slot.id == "broad") + .expect("should keep broad slot"); + assert_eq!( + broad.page_patterns, + ["/broad/*"], + "shorter prefix should not claim the discovered div" + ); + let atf = merged + .iter() + .find(|slot| slot.id == "atf") + .expect("should keep specific slot"); + assert_eq!( + atf.page_patterns, + ["/", "/news/*"], + "longest matching prefix should receive this run's pattern" + ); + } + + #[test] + fn merge_renames_new_slot_id_that_collides_with_existing_config() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header-main\"\ndiv_id = \"legacy-header\"\n\ + gam_unit_path = \"/222/legacy\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "div-gpt-ad-header.main".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + let ids = merged + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + #[test] fn merge_keeps_existing_only_slots() { // Existing has header + sidebar; this run re-sees only header. diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index b7ed91f1e..1a33b890a 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -3,7 +3,7 @@ //! `ts audit page ` is the generic page audit; `ts audit ad-templates verify //! ...` is the ad-template verifier; `ts audit generate ` bootstraps a //! draft config from a live page (issue #800). `ts audit ` is a hidden -//! compatibility alias for `ts audit page `. +//! compatibility alias for `ts audit generate `. pub mod ad_templates; pub mod browser; @@ -55,9 +55,40 @@ pub(crate) fn parse_cookie(raw: &str) -> Result<(String, String), String> { pub(crate) struct AuditArgs { #[command(subcommand)] pub(crate) command: Option, - /// Hidden compatibility alias: `ts audit ` behaves like `ts audit page `. + /// Hidden compatibility alias: `ts audit ` behaves like `ts audit generate `. #[arg(value_parser = parse_http_url, hide = true)] pub(crate) legacy_url: Option, + #[command(flatten)] + pub(crate) legacy_generate: LegacyGenerateArgs, +} + +/// Hidden generation flags retained for the legacy `ts audit ` form. +#[derive(Debug, Default, Args)] +pub(crate) struct LegacyGenerateArgs { + /// JavaScript asset audit output path. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) js_assets: Option, + /// Draft Trusted Server config output path. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) config: Option, + /// Do not write the JavaScript asset audit file. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) no_js_assets: bool, + /// Do not write the draft Trusted Server config file. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) no_config: bool, + /// Overwrite existing output files. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) force: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + #[arg( + long = "cookie", + value_name = "NAME=VALUE", + value_parser = parse_cookie, + hide = true, + requires = "legacy_url" + )] + pub(crate) cookies: Vec<(String, String)>, } /// `ts audit` subcommands. @@ -136,8 +167,8 @@ pub(crate) struct AuditAdTemplatesVerifyArgs { /// Dispatches a `ts audit` invocation. /// -/// `legacy_url` (if present) and the `page` subcommand both route to the generic -/// page audit; `ad-templates verify` routes to the verifier. +/// `legacy_url` (if present) routes to artifact generation, while the `page` +/// subcommand routes to the generic read-only page audit. /// /// # Errors /// @@ -147,7 +178,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { match &args.command { Some(AuditSubcommand::Page(page_args)) => page::run_page(page_args), Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { - let loaded = crate::app_config::load_settings(&gen_args.config)?; + let loaded = crate::app_config::load_file_settings(&gen_args.config)?; let collector = generate::browser_collector::BrowserAuditCollector; let stdout = std::io::stdout(); let mut out = stdout.lock(); @@ -173,12 +204,32 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { generate::run_generate(generate_args, &collector, &mut out) } None => match &args.legacy_url { - Some(url) => page::run_page_url(url, false), + Some(_) => { + let generate_args = legacy_generate_args(args) + .expect("should build generation args when legacy URL is present"); + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let collector = generate::browser_collector::BrowserAuditCollector; + generate::run_generate(&generate_args, &collector, &mut out) + } None => Err("provide a URL or a subcommand (`page`, `ad-templates`)".to_string()), }, } } +fn legacy_generate_args(args: &AuditArgs) -> Option { + let url = args.legacy_url.as_ref()?; + Some(generate::GenerateArgs { + url: url.to_string(), + js_assets: args.legacy_generate.js_assets.clone(), + config: args.legacy_generate.config.clone(), + no_js_assets: args.legacy_generate.no_js_assets, + no_config: args.legacy_generate.no_config, + force: args.legacy_generate.force, + cookies: args.legacy_generate.cookies.clone(), + }) +} + #[cfg(test)] mod tests { use super::*; @@ -214,4 +265,39 @@ mod tests { let err = parse_cookie("=value").expect_err("should reject empty name"); assert!(err.contains("empty name"), "error should name the problem"); } + + #[test] + fn legacy_url_builds_artifact_generation_args() { + let args = AuditArgs { + command: None, + legacy_url: Some( + url::Url::parse("https://www.example.com/").expect("should parse URL"), + ), + legacy_generate: LegacyGenerateArgs { + js_assets: Some("audit/assets.toml".into()), + config: Some("audit/config.toml".into()), + no_js_assets: false, + no_config: false, + force: true, + cookies: vec![("session".to_string(), "example".to_string())], + }, + }; + + let generate = legacy_generate_args(&args).expect("should build generation args"); + + assert_eq!(generate.url, "https://www.example.com/"); + assert_eq!( + generate.js_assets.as_deref(), + Some(std::path::Path::new("audit/assets.toml")) + ); + assert_eq!( + generate.config.as_deref(), + Some(std::path::Path::new("audit/config.toml")) + ); + assert!(generate.force); + assert_eq!( + generate.cookies, + [("session".to_string(), "example".to_string())] + ); + } } diff --git a/crates/trusted-server-cli/src/commands/audit/page.rs b/crates/trusted-server-cli/src/commands/audit/page.rs index 3b511edc1..af0144fe9 100644 --- a/crates/trusted-server-cli/src/commands/audit/page.rs +++ b/crates/trusted-server-cli/src/commands/audit/page.rs @@ -35,16 +35,6 @@ pub(crate) fn run_page(args: &PageAuditArgs) -> Result<(), String> { ) } -/// Runs the generic page audit for a single URL with default browser options -/// (the legacy `ts audit ` alias entry point). -/// -/// # Errors -/// -/// Returns a user-facing string when the browser cannot collect the page. -pub(crate) fn run_page_url(url: &url::Url, scroll: bool) -> Result<(), String> { - run_with_collector(&BrowserCollector::new(), url, scroll) -} - fn run_with_collector( collector: &BrowserCollector, url: &url::Url, diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 8164533f4..b98ed3b6a 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -286,9 +286,35 @@ mod tests { } #[test] - fn audit_legacy_url_parses_as_page_alias() { - let args = parse(&["ts", "audit", "https://www.example.com/"]); - assert!(matches!(args.command, Command::Audit(_))); + fn audit_legacy_url_parses_with_artifact_generation_flags() { + let args = parse(&[ + "ts", + "audit", + "https://www.example.com/", + "--js-assets", + "audit/assets.toml", + "--config", + "audit/config.toml", + "--force", + "--cookie", + "session=example", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + assert_eq!( + audit.legacy_generate.js_assets, + Some(PathBuf::from("audit/assets.toml")) + ); + assert_eq!( + audit.legacy_generate.config, + Some(PathBuf::from("audit/config.toml")) + ); + assert!(audit.legacy_generate.force); + assert_eq!( + audit.legacy_generate.cookies, + [("session".to_string(), "example".to_string())] + ); } #[test] diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 3ef29fcec..e39afe88a 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -79,7 +79,7 @@ Chrome or Chromium must be installed locally. The command checks common PATH names and standard macOS/Linux install locations. ```bash -ts audit https://publisher.example +ts audit generate https://publisher.example ``` By default, the command writes: @@ -99,13 +99,13 @@ ts config validate If a config already exists, avoid overwriting it: ```bash -ts audit https://publisher.example --no-config +ts audit generate https://publisher.example --no-config ``` Use custom output paths when reviewing artifacts first: ```bash -ts audit https://publisher.example \ +ts audit generate https://publisher.example \ --js-assets audit/js-assets.toml \ --config audit/trusted-server.toml ``` @@ -113,9 +113,12 @@ ts audit https://publisher.example \ Use `--force` only when replacing existing output files is intentional: ```bash -ts audit https://publisher.example --force +ts audit generate https://publisher.example --force ``` +The legacy `ts audit ` form remains a compatibility alias for artifact +generation. New automation should use `ts audit generate `. + `ts audit` is not an EdgeZero adapter command. It has no `--adapter` option and it does not provision resources, push config, build, deploy, or contact platform APIs. diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 893c5b18f..0d4ab708c 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -113,7 +113,7 @@ ts config init To bootstrap from a public publisher page, run an audit first: ```bash -ts audit https://publisher.example +ts audit generate https://publisher.example ``` The audit command writes `js-assets.toml` plus a draft `trusted-server.toml`. diff --git a/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md index db8197760..018f7f685 100644 --- a/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md +++ b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md @@ -792,21 +792,23 @@ should become a subcommand namespace: ```bash ts audit page +ts audit generate ts audit ad-templates verify ... ``` The existing #800 `ts audit ` behavior should be preserved as a -compatibility alias for `ts audit page ` during the transition. New -ad-template work should use the nested namespace only. This avoids routing -ambiguity in Clap and keeps generic page audit behavior separate from -ad-template verification. +compatibility alias for `ts audit generate ` during the transition, +including its artifact output flags. This avoids a successful but silent +behavior change for existing onboarding scripts. Parsing contract: - `ts audit page ` is the canonical generic page-audit command. +- `ts audit generate ` is the canonical artifact-generation command. - `ts audit ad-templates verify ...` is the canonical ad-template verifier. -- `ts audit ` is a hidden compatibility alias for `ts audit page ` and - is accepted only when `` parses as `http` or `https`. +- `ts audit ` is a hidden compatibility alias for + `ts audit generate ` and is accepted only when `` parses as `http` + or `https`. - `ts audit ad-templates` must never be treated as a legacy URL positional. - `ts audit page` without a URL must fail with the normal Clap missing-argument error. @@ -834,7 +836,7 @@ If Clap cannot enforce the optional-subcommand plus hidden positional contract cleanly, implement a small custom dispatcher for the `audit` argv tail and test it directly. Required parser tests: -- `ts audit https://www.example.com/` dispatches to page audit; +- `ts audit https://www.example.com/` dispatches to artifact generation; - `ts audit page https://www.example.com/` dispatches to page audit; - `ts audit ad-templates verify https://www.example.com/` dispatches to ad-template verification; From c0da7e7fa58e6be53d9e26f373586567e65995d3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 9 Jul 2026 16:22:09 +0530 Subject: [PATCH 139/395] Box audit CLI arguments --- crates/trusted-server-cli/src/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index b98ed3b6a..9aade197e 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -24,7 +24,7 @@ enum Command { /// Sign in / out / status against an `EdgeZero` adapter. Auth(AuthArgs), /// Browser-backed page and ad-template audits. - Audit(AuditArgs), + Audit(Box), /// Build the project for a target adapter. Build(BuildArgs), /// Trusted Server app-config commands. From 39d22ca3c669f303493b8dceaf84f27fc6a746cd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 9 Jul 2026 16:58:59 +0530 Subject: [PATCH 140/395] Run browser fixture tests serially --- .github/workflows/test.yml | 3 +-- crates/trusted-server-cli/src/commands/audit/browser.rs | 4 ++-- scripts/test-cli.sh | 2 ++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5f2717dcb..8ed0a920b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -210,8 +210,7 @@ jobs: cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings - name: cargo test - run: | - cargo test --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" + run: ./scripts/test-cli.sh test-typescript: name: vitest diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs index 9f6aca1e6..a67591b59 100644 --- a/crates/trusted-server-cli/src/commands/audit/browser.rs +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -489,12 +489,12 @@ mod tests { "#; #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] fn collects_gpt_slot_from_local_fixture() { if !chrome_available() { // Browser fixture test requires a local Chrome/Chromium; skipping. return; } - let mut fixture = tempfile::Builder::new() .suffix(".html") .tempfile() @@ -536,12 +536,12 @@ mod tests { } #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] fn scroll_pass_keeps_initial_load_phase_for_load_time_evidence() { if !chrome_available() { // Browser fixture test requires a local Chrome/Chromium; skipping. return; } - let mut fixture = tempfile::Builder::new() .suffix(".html") .tempfile() diff --git a/scripts/test-cli.sh b/scripts/test-cli.sh index eef9e2f7d..379771675 100755 --- a/scripts/test-cli.sh +++ b/scripts/test-cli.sh @@ -19,3 +19,5 @@ if ! rustup target list --installed | awk -v target="$HOST_TARGET" '$0 == target fi cargo test --package trusted-server-cli --target "$HOST_TARGET" +cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + commands::audit::browser::tests:: -- --ignored --test-threads=1 From 671a742eac0d011fe431b9052baa08680ecf3260 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 15 Jul 2026 16:22:31 +0530 Subject: [PATCH 141/395] Make ad-template audit collector resilient to non-loading pages Ad-heavy publisher pages (video players, continuous ad refresh, anti-bot scripts) may never fire the `load` event, so `page.goto` would block until the navigation timeout and the audit failed before scraping any slots. Article pages consistently timed out this way while lighter listing pages succeeded. Navigate without hard-failing on the load wait: a load-wait or main-document-response timeout is downgraded to a "results may be partial" warning, and the existing settle loop is the real readiness signal. The settle loop now also accepts `interactive` readyState, since these pages define their GPT slots before (or without ever) reaching `complete`. Load wait is bounded separately at 12s and the settle cap is raised to 12s so lazily-defined slots are captured. --- .../audit/generate/browser_collector.rs | 66 ++++++++++++------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index 1f0694bf4..b1446933c 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -19,8 +19,13 @@ use crate::error::{CliResult, report_error}; const SETTLE_QUIET_PERIOD: Duration = Duration::from_millis(750); const SETTLE_POLL_INTERVAL: Duration = Duration::from_millis(250); -const SETTLE_MAX_WAIT: Duration = Duration::from_secs(6); -const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); +const SETTLE_MAX_WAIT: Duration = Duration::from_secs(12); +/// How long to wait for the navigation `load` event (and, separately, the main +/// document response) before falling through to the settle loop. Ad-heavy pages +/// (video players, continuous ad refresh) may never fire `load`, so this is a +/// soft bound: the settle loop is the real readiness signal and the scrape reads +/// whatever rendered by then. +const NAVIGATION_LOAD_TIMEOUT: Duration = Duration::from_secs(12); const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; const RESOURCE_TIMING_BUFFER_WARNING: &str = @@ -124,28 +129,41 @@ async fn collect_page_from_browser( .map_err(|error| report_error(format!("failed to set cookie `{name}`: {error}")))?; } - timeout(NAVIGATION_TIMEOUT, page.goto(target_url.as_str())) - .await - .map_err(|_| report_error(format!("timed out navigating to `{target_url}`")))? - .map_err(|error| report_error(format!("failed to navigate to `{target_url}`: {error}")))?; + let mut warnings = Vec::new(); - let navigation_response = timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation_response()) - .await - .map_err(|_| { - report_error(format!( - "timed out waiting for main document navigation response from `{target_url}`" - )) - })? - .map_err(|error| { - report_error(format!( - "failed to read main document navigation response: {error}" - )) - })?; + // Navigate, but don't hard-fail when the `load` event never fires. Ad-heavy + // pages (video players, continuous ad refresh, anti-bot scripts) can keep + // the frame "loading" indefinitely, so a load-wait timeout is downgraded to + // a warning: the settle loop below is the real readiness signal and the + // scrape reads whatever rendered by then. + match timeout(NAVIGATION_LOAD_TIMEOUT, page.goto(target_url.as_str())).await { + Ok(Ok(_)) => {} + Ok(Err(error)) => warnings.push(format!( + "navigation to `{target_url}` did not complete cleanly ({error}); results may be partial" + )), + Err(_) => warnings.push(format!( + "navigation to `{target_url}` did not fire `load` within {}s; results may be partial", + NAVIGATION_LOAD_TIMEOUT.as_secs() + )), + } - let mut warnings = Vec::new(); - if let Some(warning) = validate_navigation_response(navigation_response)? { - warnings.push(warning); + // Best-effort read of the main-document response for status validation. When + // the load wait above times out the response is usually already buffered, so + // this returns promptly; tolerate a miss rather than failing the audit. + match timeout(NAVIGATION_LOAD_TIMEOUT, page.wait_for_navigation_response()).await { + Ok(Ok(navigation_response)) => { + if let Some(warning) = validate_navigation_response(navigation_response)? { + warnings.push(warning); + } + } + Ok(Err(error)) => warnings.push(format!( + "could not read the main document response from `{target_url}` ({error}); results may be partial" + )), + Err(_) => warnings.push(format!( + "timed out reading the main document response from `{target_url}`; results may be partial" + )), } + if !wait_for_page_settle(&page).await? { warnings.push( "browser audit timed out while waiting for the page to settle; results may be partial" @@ -286,7 +304,11 @@ async fn wait_for_page_settle(page: &chromiumoxide::Page) -> CliResult { .into_value() .map_err(|error| report_error(format!("failed to decode resource count: {error}")))?; - if ready_state == "complete" { + // Accept `interactive` as well as `complete`: ad-heavy pages often never + // reach `complete` (the `load` event never fires), but their GPT slots + // are defined once the DOM is interactive, so a quiet network period at + // `interactive` is a valid settle signal for the slot scrape. + if ready_state == "complete" || ready_state == "interactive" { if previous_count == Some(resource_count) { stable_for += SETTLE_POLL_INTERVAL; } else { From 1d852a7c6115b42978f5a6d1ceeb83c607f2ca28 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 15 Jul 2026 17:22:53 +0530 Subject: [PATCH 142/395] Keep one managed-slot header comment across generate re-runs `render_slots` prepends a `# Slots managed by ...` header, but the in-place splice preserved the previous copy in the scalar block and inserted a fresh one, so each `ts audit ad-templates generate` run against an already-managed config appended another duplicate comment block. Extract the two header lines to constants and strip any prior copy (and the blank lines it leaves) from the preserved head before re-inserting the rendered slots, so repeated runs keep exactly one header. Add a regression test that splices three times and asserts a single header. --- .../src/commands/audit/generate/slot_toml.rs | 62 +++++++++++++++++-- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index b15e6ed9f..63f81b50c 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -188,12 +188,17 @@ fn matching_slot_index(existing: &[RenderSlot], discovered: &RenderSlot) -> Opti existing.iter().position(|slot| slot.key() == key) } +/// Header comment emitted above the managed slot array. Stripped from the +/// preserved scalar block on re-splice (see [`is_managed_comment_line`]) so +/// repeated `generate` runs don't accumulate duplicate copies. +const MANAGED_SLOTS_COMMENT: &str = "# Slots managed by `ts audit ad-templates generate`."; +/// Second line of the managed-slot header comment. +const MANAGED_SLOTS_REVIEW_COMMENT: &str = + "# Review page_patterns and formats before validating/pushing."; + /// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. pub(super) fn render_slots(slots: &[RenderSlot]) -> String { - let mut out = String::from( - "\n# Slots managed by `ts audit ad-templates generate`.\n\ - # Review page_patterns and formats before validating/pushing.\n", - ); + let mut out = format!("\n{MANAGED_SLOTS_COMMENT}\n{MANAGED_SLOTS_REVIEW_COMMENT}\n"); for slot in slots { out.push_str("\n[[creative_opportunities.slot]]\n"); out.push_str(&format!("id = {}\n", toml_string(&slot.id))); @@ -409,9 +414,20 @@ pub(super) fn splice_creative_slots( .position(|line| is_unrelated_table(line)) .map_or(lines.len(), |offset| start + offset); - let mut result = lines[..start].join("\n"); + // Preserve everything before the slot array, but drop any prior managed + // header comment (and the blank lines it leaves behind): `rendered` re-emits + // it, so keeping the old copy would duplicate it on every re-splice. + let mut head_lines: Vec<&str> = lines[..start] + .iter() + .copied() + .filter(|line| !is_managed_comment_line(line)) + .collect(); + while head_lines.last().is_some_and(|line| line.trim().is_empty()) { + head_lines.pop(); + } + let mut result = head_lines.join("\n"); if !result.is_empty() { - result.push('\n'); + result.push_str("\n\n"); } result.push_str(rendered); result.push('\n'); @@ -478,6 +494,14 @@ fn is_table_header(line: &str, section_header: &str) -> bool { strip_inline_comment(line.trim()) == section_header } +/// Whether `line` is one of the managed header comment lines emitted by +/// [`render_slots`]. Used to strip the prior copy on re-splice so repeated +/// `generate` runs keep exactly one header comment. +fn is_managed_comment_line(line: &str) -> bool { + let trimmed = line.trim(); + trimmed == MANAGED_SLOTS_COMMENT || trimmed == MANAGED_SLOTS_REVIEW_COMMENT +} + pub(super) fn replace_key_in_section( document: &str, section: &str, @@ -681,6 +705,32 @@ mod tests { ); } + #[test] + fn resplice_does_not_accumulate_managed_comment() { + // A re-run splices into a config that already carries the managed + // header comment; it must keep exactly one copy, not append another. + let first = splice_creative_slots( + "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n", + Some("222"), + &header_rendered(), + ) + .expect("first splice"); + let second = + splice_creative_slots(&first, Some("222"), &header_rendered()).expect("second splice"); + let third = + splice_creative_slots(&second, Some("222"), &header_rendered()).expect("third splice"); + + assert_eq!( + third + .lines() + .filter(|line| line.trim() == MANAGED_SLOTS_COMMENT) + .count(), + 1, + "managed header comment must not accumulate across re-splices" + ); + toml::from_str::(&third).expect("re-spliced config stays valid TOML"); + } + #[test] fn splice_recognizes_inline_commented_section_header() { // `[creative_opportunities] # comment` is valid TOML; the splice must From e081e2c166ec3c8743e09cb790010b40e18f9ae2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 15 Jul 2026 17:36:29 +0530 Subject: [PATCH 143/395] Wrap assert! in ad-stack gate test to satisfy CI rustfmt CI's rustfmt wraps the single method-chain argument of this assert! onto its own lines; the compact form the merge brought in passed locally but failed the format gate. Match CI's canonical form. --- crates/trusted-server-core/src/creative_opportunities.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index b643bb914..e55c5676f 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -594,9 +594,11 @@ mod tests { }); assert_eq!(result.expected, RuntimeAdStackExpected::No); - assert!(result - .blocking_gates() - .contains(&AdStackGateName::AuctionEnabled)); + assert!( + result + .blocking_gates() + .contains(&AdStackGateName::AuctionEnabled) + ); } #[test] From d5be3d96b86f4d6c693d219af54d5fe6b6e674a9 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 17 Jul 2026 18:01:32 +0530 Subject: [PATCH 144/395] Add admin endpoint to look up EC entries by id Adds GET /_ts/admin/ec/{id} (explicit EC ID) and GET /_ts/admin/ec (EC ID from the caller's ts-ec cookie) so operators can inspect EC identity graph entries and debug KV-to-auction EID propagation. The core handler returns the stored KvEntry verbatim (including raw consent strings and partner UIDs), the KV metadata mirror, the store generation marker, and a derived auction view showing exactly which EIDs the auction would attach and why each stored partner ID was skipped (empty_uid, not_in_registry, bidstream_disabled). Corrupt entries are returned with the parse error and raw body via the new KvIdentityGraph::lookup_raw instead of failing closed. The routes join Settings::ADMIN_ENDPOINTS so startup validation rejects configs whose basic-auth handler regex does not cover them. The EC identity graph is Fastly KV backed, so the Axum, Cloudflare, and Spin adapters register the routes to local 501 responses, keeping them off the publisher fallback that would forward the Authorization header to the origin. Closes #921 --- crates/trusted-server-adapter-axum/src/app.rs | 31 +- .../tests/routes.rs | 34 + .../src/app.rs | 25 + .../tests/routes.rs | 28 + .../trusted-server-adapter-fastly/src/app.rs | 44 ++ crates/trusted-server-adapter-spin/src/app.rs | 29 +- .../tests/routes.rs | 26 + crates/trusted-server-core/src/ec/admin.rs | 626 ++++++++++++++++++ crates/trusted-server-core/src/ec/kv.rs | 21 +- crates/trusted-server-core/src/ec/mod.rs | 1 + crates/trusted-server-core/src/settings.rs | 28 +- 11 files changed, 885 insertions(+), 8 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/admin.rs diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 2f4329574..12acbfc68 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -252,6 +252,7 @@ enum NamedRouteHandler { TrustedServerDiscovery, VerifySignature, AdminNotSupported, + AdminEcNotSupported, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, @@ -279,7 +280,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_routes() -> [NamedRoute; 12] { +fn named_routes() -> [NamedRoute; 14] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -304,6 +305,19 @@ fn named_routes() -> [NamedRoute; 12] { primary_methods: &[Method::POST], handler: NamedRouteHandler::AdminNotSupported, }, + // Admin EC lookup routes. Registered explicitly (like the key routes + // above) so they never fall through to the publisher fallback, and + // they match `Settings::ADMIN_ENDPOINTS` for auth coverage. + NamedRoute { + path: "/_ts/admin/ec", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcNotSupported, + }, + NamedRoute { + path: "/_ts/admin/ec/{id}", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcNotSupported, + }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with // a 404, matching the Fastly and Cloudflare adapters: the production // basic-auth handler regex `^/_ts/admin` does not match them, and letting @@ -388,6 +402,21 @@ fn named_route_handler( ); Ok(resp) } + NamedRouteHandler::AdminEcNotSupported => { + // The EC identity graph is Fastly KV backed; the Axum + // dev server has no store to read. + let body = edgezero_core::body::Body::from( + "Admin EC lookup is not supported on the Axum dev server.\n\ + Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", + ); + let mut resp = Response::new(body); + *resp.status_mut() = StatusCode::NOT_IMPLEMENTED; + resp.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + Ok(resp) + } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::Auction => { // Build the geo-aware EC context so the auction consent diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index c4bf7d990..7c20e2dd2 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -74,6 +74,8 @@ fn all_explicit_routes_are_registered() { ("POST", "/verify-signature"), ("POST", "/_ts/admin/keys/rotate"), ("POST", "/_ts/admin/keys/deactivate"), + ("GET", "/_ts/admin/ec"), + ("GET", "/_ts/admin/ec/{id}"), ("POST", "/admin/keys/rotate"), ("POST", "/admin/keys/deactivate"), ("POST", "/auction"), @@ -256,6 +258,38 @@ async fn admin_route_without_credentials_returns_401() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_ec_routes_return_501() { + // The EC identity graph is Fastly KV backed, so the Axum dev server + // answers the admin EC lookup routes locally with 501 instead of letting + // them fall through to the publisher fallback. + let sample_ec_id = format!("{}.abc123", "a".repeat(64)); + for path in [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{sample_ec_id}"), + ] { + let mut svc = make_service(); + let req = Request::builder() + .method("GET") + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(AxumBody::empty()) + .expect("should build request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + assert_eq!( + resp.status().as_u16(), + 501, + "{path} should report that Axum EC lookup is unsupported" + ); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { // Regression for the credential-leak finding: the production basic-auth regex diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index c931360f6..85eb09f1b 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -242,6 +242,20 @@ fn admin_key_management_not_supported() -> Response { response } +fn admin_ec_lookup_not_supported() -> Response { + let body = edgezero_core::body::Body::from( + "Admin EC lookup is not supported on Cloudflare Workers.\n\ + Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", + ); + let mut response = Response::new(body); + *response.status_mut() = StatusCode::NOT_IMPLEMENTED; + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + response +} + /// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` /// aliases on the Cloudflare adapter. /// @@ -461,6 +475,17 @@ fn build_router(state: &Arc) -> RouterService { .post("/_ts/admin/keys/deactivate", |_ctx: RequestContext| async { Ok::(admin_key_management_not_supported()) }) + // Admin EC lookup routes. Registered explicitly (like the key + // routes above) so they never fall through to the publisher + // fallback, and they match `Settings::ADMIN_ENDPOINTS` for auth + // coverage. The EC identity graph is Fastly KV backed, so this + // adapter has no store to read. + .get("/_ts/admin/ec", |_ctx: RequestContext| async { + Ok::(admin_ec_lookup_not_supported()) + }) + .get("/_ts/admin/ec/{id}", |_ctx: RequestContext| async { + Ok::(admin_ec_lookup_not_supported()) + }) .post( "/auction", make_handler(Arc::clone(&state), |s, services, req| async move { diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index df2781945..7b048833b 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -215,6 +215,8 @@ fn all_explicit_routes_are_registered() { ("POST", "/verify-signature"), ("POST", "/_ts/admin/keys/rotate"), ("POST", "/_ts/admin/keys/deactivate"), + ("GET", "/_ts/admin/ec"), + ("GET", "/_ts/admin/ec/{id}"), ("POST", "/auction"), ("GET", "/first-party/proxy"), ("GET", "/first-party/click"), @@ -264,6 +266,32 @@ async fn authenticated_admin_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_ec_routes_return_501() { + // The EC identity graph is Fastly KV backed, so Cloudflare answers the + // admin EC lookup routes locally with 501 instead of letting them fall + // through to the publisher fallback. + let sample_ec_id = format!("{}.abc123", "a".repeat(64)); + for path in [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{sample_ec_id}"), + ] { + let req = request_builder() + .method("GET") + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 501, + "{path} should report that Cloudflare EC lookup is unsupported" + ); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_route_without_credentials_returns_401() { let router = test_router(); diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 955ff235b..71d906d52 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -22,6 +22,8 @@ //! | POST | `/verify-signature` | [`handle_verify_signature`] | //! | POST | `/_ts/admin/keys/rotate` | [`handle_rotate_key`] | //! | POST | `/_ts/admin/keys/deactivate` | [`handle_deactivate_key`] | +//! | GET | `/_ts/admin/ec` | [`handle_admin_ec_lookup`] | +//! | GET | `/_ts/admin/ec/{id}` | [`handle_admin_ec_lookup`] | //! | POST | `/_ts/api/v1/batch-sync` | [`handle_batch_sync`] | //! | GET | `/_ts/api/v1/identify` | [`handle_identify`] | //! | GET | `/_ts/set-tester` | [`handle_set_tester`] | @@ -98,6 +100,7 @@ use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::handle_admin_ec_lookup; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; @@ -565,6 +568,10 @@ async fn run_named_route( } NamedRouteHandler::RotateKey => handle_rotate_key(&state.settings, services, req), NamedRouteHandler::DeactivateKey => handle_deactivate_key(&state.settings, services, req), + NamedRouteHandler::AdminEcLookup => { + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + handle_admin_ec_lookup(ec.kv_graph.as_ref(), &partner_registry, &req) + } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::BatchSync => { // Dispatched by execute_named before EC state is built. @@ -987,6 +994,7 @@ enum NamedRouteHandler { VerifySignature, RotateKey, DeactivateKey, + AdminEcLookup, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, @@ -1039,6 +1047,18 @@ const NAMED_ROUTES: &[NamedRoute] = &[ primary_methods: &[Method::POST], handler: NamedRouteHandler::DeactivateKey, }, + // Admin EC lookup: the bare route reads the EC ID from the caller's + // `ts-ec` cookie; the parameterized route takes an explicit EC ID. + NamedRoute { + path: "/_ts/admin/ec", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcLookup, + }, + NamedRoute { + path: "/_ts/admin/ec/{id}", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcLookup, + }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with a // 404 instead of executing key operations: the production basic-auth handler // regex `^/_ts/admin` does not match them, and letting them fall through to @@ -1624,6 +1644,30 @@ mod tests { } } + #[test] + fn admin_ec_lookup_routes_are_registered() { + // Both lookup shapes must be explicitly routed to the admin EC + // handler: the bare cookie-based route and the parameterized route. + // Leaving either unrouted would fall through to the publisher + // fallback, forwarding the caller's `Authorization` header to the + // origin. + for path in ["/_ts/admin/ec", "/_ts/admin/ec/{id}"] { + let route = NAMED_ROUTES + .iter() + .find(|route| route.path == path) + .unwrap_or_else(|| panic!("{path} must be a named route")); + assert!( + matches!(route.handler, NamedRouteHandler::AdminEcLookup), + "{path} must map to the admin EC lookup handler" + ); + assert_eq!( + route.primary_methods, + &[Method::GET], + "{path} must have GET as its only primary method" + ); + } + } + #[test] fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { // Regression for the credential-leak finding: with a production-shaped diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 2291fce74..29ca574ff 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -141,12 +141,14 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 12] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 14] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), ("/_ts/admin/keys/rotate", &[Method::POST]), ("/_ts/admin/keys/deactivate", &[Method::POST]), + ("/_ts/admin/ec", &[Method::GET]), + ("/_ts/admin/ec/{id}", &[Method::GET]), ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), ("/auction", &[Method::POST]), @@ -359,6 +361,20 @@ fn admin_key_management_not_supported() -> Response { response } +fn admin_ec_lookup_not_supported() -> Response { + let body = edgezero_core::body::Body::from( + "Admin EC lookup is not supported on Fermyon Spin.\n\ + Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", + ); + let mut response = Response::new(body); + *response.status_mut() = StatusCode::NOT_IMPLEMENTED; + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + response +} + // --------------------------------------------------------------------------- // Error helper // --------------------------------------------------------------------------- @@ -511,6 +527,10 @@ fn build_router(state: &Arc) -> RouterService { Ok::(admin_key_management_not_supported()) }; + let admin_ec_not_supported_handler = |_ctx: RequestContext| async { + Ok::(admin_ec_lookup_not_supported()) + }; + // /auction let s = Arc::clone(&state); let auction_handler = move |ctx: RequestContext| { @@ -730,6 +750,13 @@ fn build_router(state: &Arc) -> RouterService { // credentials and key-management payloads to the origin. .post("/_ts/admin/keys/rotate", admin_not_supported_handler) .post("/_ts/admin/keys/deactivate", admin_not_supported_handler) + // Admin EC lookup routes. Registered explicitly (like the key + // routes above) so they never fall through to the publisher + // fallback, and they match `Settings::ADMIN_ENDPOINTS` for auth + // coverage. The EC identity graph is Fastly KV backed, so this + // adapter has no store to read. + .get("/_ts/admin/ec", admin_ec_not_supported_handler) + .get("/_ts/admin/ec/{id}", admin_ec_not_supported_handler) .post("/auction", auction_handler) .get("/__ts/page-bids", page_bids_handler) .route( diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 9b96dbd70..4194baea4 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -113,6 +113,32 @@ async fn authenticated_admin_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_ec_routes_return_501() { + // The EC identity graph is Fastly KV backed, so Spin answers the admin + // EC lookup routes locally with 501 instead of letting them fall through + // to the publisher fallback. + let sample_ec_id = format!("{}.abc123", "a".repeat(64)); + for path in [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{sample_ec_id}"), + ] { + let req = request_builder() + .method("GET") + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 501, + "{path} should report that Spin EC lookup is unsupported" + ); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn health_route_returns_ok() { // Parity with the Fastly/Axum adapters: GET /health is a cheap liveness probe diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs new file mode 100644 index 000000000..54599d59d --- /dev/null +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -0,0 +1,626 @@ +//! Admin endpoint for inspecting EC identity graph entries. +//! +//! Serves `GET /_ts/admin/ec` (EC ID taken from the request's `ts-ec` +//! cookie) and `GET /_ts/admin/ec/{id}` (explicit EC ID). Returns the raw +//! stored [`KvEntry`] plus a derived view of the EIDs the auction would +//! attach, so operators can debug KV-to-auction propagation without KV +//! console access. +//! +//! Authentication is enforced by the `^/_ts/admin` basic-auth handler +//! configuration; startup validation rejects configs that leave these paths +//! uncovered (see `Settings::ADMIN_ENDPOINTS`). Because the endpoint is +//! auth-gated and operator-facing, responses intentionally include full +//! internal detail (raw consent strings, partner UIDs, parse errors). + +use http::{Request, Response, StatusCode, header}; +use serde::Serialize; +use serde_json::Value as JsonValue; + +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; + +use crate::constants::COOKIE_TS_EC; +use crate::error::TrustedServerError; +use crate::openrtb::Eid; + +use super::eids::{resolve_partner_ids, to_eids}; +use super::generation::is_valid_ec_id; +use super::kv::KvIdentityGraph; +use super::kv_backend::EcKvLookup; +use super::kv_types::{KvEntry, KvMetadata}; +use super::log_id; +use super::registry::PartnerRegistry; + +/// Route prefix shared by the cookie-based and explicit-ID lookup routes. +const ADMIN_EC_PATH: &str = "/_ts/admin/ec"; + +/// Successful admin EC lookup payload. +#[derive(Debug, Serialize)] +struct AdminEcLookupResponse { + /// The EC ID that was looked up. + ec_id: String, + /// Platform KV store name the entry was read from. + store: String, + /// Store generation marker for the entry. + generation: u64, + /// `true` when the entry is a consent-withdrawal tombstone + /// (`consent.ok = false`). Absent when the body failed to parse. + #[serde(skip_serializing_if = "Option::is_none")] + tombstone: Option, + /// The stored entry, re-serialized verbatim. Absent when the body + /// failed to deserialize (see `entry_error` / `raw_body`). + #[serde(skip_serializing_if = "Option::is_none")] + entry: Option, + /// Deserialization or validation failure detail for the entry body. + #[serde(skip_serializing_if = "Option::is_none")] + entry_error: Option, + /// Raw entry body (lossy UTF-8) when it could not be deserialized. + #[serde(skip_serializing_if = "Option::is_none")] + raw_body: Option, + /// The stored KV metadata mirror, when present and parseable. + #[serde(skip_serializing_if = "Option::is_none")] + metadata: Option, + /// Deserialization failure detail for the metadata, including its raw + /// value. + #[serde(skip_serializing_if = "Option::is_none")] + metadata_error: Option, + /// Derived auction view. Present only when the entry deserializes and + /// validates — the same precondition the auction read path applies, so + /// its absence means the auction would attach no KV-derived EIDs. + /// Live requests additionally gate on per-request consent, which is not + /// reproducible here. + #[serde(skip_serializing_if = "Option::is_none")] + auction: Option, +} + +/// What the auction EID decoration would produce for this entry. +#[derive(Debug, Serialize)] +struct AuctionEidsView { + /// EIDs the auction would attach to `user.eids`, exactly as produced by + /// the auction resolution path. + eids: Vec, + /// Stored partner IDs that the auction resolution filters out, with the + /// reason each was skipped. + skipped: Vec, +} + +/// A stored partner ID excluded from auction EIDs. +#[derive(Debug, Serialize)] +struct SkippedPartnerId { + /// Partner namespace key in the entry's `ids` map. + source_domain: String, + /// Why the auction resolution skips it: `empty_uid`, `not_in_registry`, + /// or `bidstream_disabled`. + reason: &'static str, +} + +/// Handles `GET /_ts/admin/ec` and `GET /_ts/admin/ec/{id}`. +/// +/// Resolves the EC ID from the path when present, falling back to the +/// request's `ts-ec` cookie for the bare route. Responds: +/// +/// - `200 OK` with an [`AdminEcLookupResponse`] JSON body when the key +/// exists (including corrupt entries, which are reported with +/// `entry_error` and `raw_body` instead of failing closed); +/// - `400 Bad Request` when the resolved ID is not a valid EC ID; +/// - `404 Not Found` when the key does not exist, or the bare route was +/// called without a `ts-ec` cookie; +/// - `501 Not Implemented` when no EC identity graph is configured. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::KvStore`] when the store open or read +/// fails. +pub fn handle_admin_ec_lookup( + kv: Option<&KvIdentityGraph>, + registry: &PartnerRegistry, + req: &Request, +) -> Result, Report> { + let Some(kv) = kv else { + return Ok(json_error( + StatusCode::NOT_IMPLEMENTED, + "EC identity graph is not configured on this deployment", + )); + }; + + let ec_id = match requested_ec_id(req) { + Ok(ec_id) => ec_id, + Err(response) => return Ok(*response), + }; + + let Some(lookup) = kv.lookup_raw(&ec_id)? else { + log::info!("Admin EC lookup: no entry for '{}'", log_id(&ec_id)); + return Ok(json_error( + StatusCode::NOT_FOUND, + "EC entry not found (KV reads are eventually consistent; a very \ + recent entry may not be visible yet)", + )); + }; + + log::info!("Admin EC lookup: returning entry for '{}'", log_id(&ec_id)); + let payload = build_lookup_response(registry, kv.store_name(), ec_id, &lookup); + let body = + serde_json::to_string(&payload).change_context(TrustedServerError::Configuration { + message: "failed to serialize admin EC lookup response".to_owned(), + })?; + Ok(json_response(StatusCode::OK, body)) +} + +/// Resolves the EC ID to look up from the path or the `ts-ec` cookie. +/// +/// Returns the (boxed) error response to send directly when no valid ID is +/// available. +fn requested_ec_id(req: &Request) -> Result>> { + let remainder = req + .uri() + .path() + .strip_prefix(ADMIN_EC_PATH) + .unwrap_or("") + .trim_matches('/'); + + let ec_id = if remainder.is_empty() { + match extract_cookie_value(req, COOKIE_TS_EC) { + Some(cookie_ec_id) => cookie_ec_id, + None => { + return Err(Box::new(json_error( + StatusCode::NOT_FOUND, + "no EC ID in path and no ts-ec cookie on the request", + ))); + } + } + } else { + remainder.to_owned() + }; + + if !is_valid_ec_id(&ec_id) { + return Err(Box::new(json_error( + StatusCode::BAD_REQUEST, + "invalid EC ID format (expected {64hex}.{6alnum})", + ))); + } + + Ok(ec_id) +} + +/// Builds the success payload from a raw KV lookup. +/// +/// Parse failures are reported in the payload rather than propagated, so +/// corrupt entries remain inspectable. +fn build_lookup_response( + registry: &PartnerRegistry, + store_name: &str, + ec_id: String, + lookup: &EcKvLookup, +) -> AdminEcLookupResponse { + let mut payload = AdminEcLookupResponse { + ec_id, + store: store_name.to_owned(), + generation: lookup.generation, + tombstone: None, + entry: None, + entry_error: None, + raw_body: None, + metadata: None, + metadata_error: None, + auction: None, + }; + + match serde_json::from_slice::(&lookup.body) { + Ok(entry) => { + payload.tombstone = Some(!entry.consent.ok); + match entry.validate() { + Ok(()) => payload.auction = Some(build_auction_view(registry, &entry)), + Err(message) => { + payload.entry_error = Some(format!( + "entry failed validation (auction reads fail closed \ + and attach no EIDs): {message}" + )); + } + } + payload.entry = Some(serde_json::to_value(&entry).expect("should serialize KvEntry")); + } + Err(error) => { + payload.entry_error = Some(format!("failed to deserialize entry: {error}")); + payload.raw_body = Some(String::from_utf8_lossy(&lookup.body).into_owned()); + } + } + + match &lookup.metadata { + None => {} + Some(bytes) => match serde_json::from_slice::(bytes) { + Ok(metadata) => { + payload.metadata = + Some(serde_json::to_value(&metadata).expect("should serialize KvMetadata")); + } + Err(error) => { + payload.metadata_error = Some(format!( + "failed to deserialize metadata: {error} (raw: {})", + String::from_utf8_lossy(bytes) + )); + } + }, + } + + payload +} + +/// Derives the auction EID view for a valid entry, mirroring the filters in +/// [`resolve_partner_ids`] and reporting why each stored ID was skipped. +fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEidsView { + let resolved = resolve_partner_ids(registry, entry); + let eids = to_eids(&resolved); + + let mut skipped = Vec::new(); + for (source_domain, partner_uid) in &entry.ids { + let reason = if partner_uid.uid.is_empty() { + "empty_uid" + } else { + match registry.get(source_domain) { + None => "not_in_registry", + Some(partner) if !partner.bidstream_enabled => "bidstream_disabled", + Some(_) => continue, + } + }; + skipped.push(SkippedPartnerId { + source_domain: source_domain.clone(), + reason, + }); + } + + AuctionEidsView { eids, skipped } +} + +fn extract_cookie_value(req: &Request, name: &str) -> Option { + let cookie_header = req + .headers() + .get(header::COOKIE) + .and_then(|value| value.to_str().ok())?; + for pair in cookie_header.split(';') { + let pair = pair.trim(); + if let Some((key, value)) = pair.split_once('=') + && key.trim() == name + { + return Some(value.trim().to_owned()); + } + } + None +} + +fn json_error(status: StatusCode, message: &str) -> Response { + let body = serde_json::json!({ "error": message }); + json_response(status, body.to_string()) +} + +fn json_response(status: StatusCode, body: String) -> Response { + Response::builder() + .status(status) + .header(header::CONTENT_TYPE, mime::APPLICATION_JSON.as_ref()) + .header(header::CACHE_CONTROL, "no-store") + .body(EdgeBody::from(body.into_bytes())) + .expect("should build admin EC lookup response") +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::ec::kv_backend::test_support::InMemoryEcKv; + use crate::ec::kv_backend::{EcKvStore as _, EcKvWrite, EcKvWriteMode}; + use crate::ec::kv_types::KvPartnerId; + use crate::redacted::Redacted; + use crate::settings::EcPartner; + + fn test_ec_id() -> String { + format!("{}.abc123", "a".repeat(64)) + } + + fn make_test_partner(source_domain: &str, bidstream_enabled: bool) -> EcPartner { + EcPartner { + name: format!("Partner {source_domain}"), + source_domain: source_domain.to_owned(), + openrtb_atype: EcPartner::default_openrtb_atype(), + bidstream_enabled, + api_token: Redacted::new(format!("test-token-{source_domain:-<32}")), + batch_rate_limit: EcPartner::default_batch_rate_limit(), + pull_sync_enabled: false, + pull_sync_url: None, + pull_sync_allowed_domains: vec![], + pull_sync_ttl_sec: EcPartner::default_pull_sync_ttl_sec(), + pull_sync_rate_limit: EcPartner::default_pull_sync_rate_limit(), + ts_pull_token: None, + } + } + + fn test_registry() -> PartnerRegistry { + PartnerRegistry::from_config(&[ + make_test_partner("bidstream.example", true), + make_test_partner("disabled.example", false), + ]) + .expect("should build test partner registry") + } + + fn get_request(path: &str) -> Request { + Request::builder() + .method("GET") + .uri(format!("https://edge.example.com{path}")) + .body(EdgeBody::empty()) + .expect("should build test request") + } + + fn get_request_with_cookie(path: &str, cookie: &str) -> Request { + Request::builder() + .method("GET") + .uri(format!("https://edge.example.com{path}")) + .header(header::COOKIE, cookie) + .body(EdgeBody::empty()) + .expect("should build test request") + } + + fn kv_with_entry(ec_id: &str, entry: &KvEntry) -> KvIdentityGraph { + let kv = KvIdentityGraph::in_memory("test-store"); + kv.create(ec_id, entry).expect("should seed KV entry"); + kv + } + + fn kv_with_raw_body(ec_id: &str, body: &str) -> KvIdentityGraph { + let metadata = serde_json::json!({ "ok": true, "country": "US", "v": 1 }).to_string(); + let store = InMemoryEcKv::new("test-store"); + store + .insert( + ec_id, + EcKvWrite { + body, + metadata: &metadata, + ttl: Duration::from_secs(60), + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed raw KV body"); + KvIdentityGraph::new(store) + } + + fn response_json(response: Response) -> JsonValue { + serde_json::from_slice(&response.into_body().into_bytes().unwrap_or_default()) + .expect("should parse response body as JSON") + } + + fn sample_entry() -> KvEntry { + let mut entry = KvEntry::minimal("bidstream.example", "uid-live", 1_741_824_000); + entry.ids.insert( + "disabled.example".to_owned(), + KvPartnerId { + uid: "uid-disabled".to_owned(), + }, + ); + entry.ids.insert( + "unknown.example".to_owned(), + KvPartnerId { + uid: "uid-unknown".to_owned(), + }, + ); + entry + } + + #[test] + fn returns_entry_with_auction_view() { + let ec_id = test_ec_id(); + let kv = kv_with_entry(&ec_id, &sample_entry()); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("no-store"), + "should send no-store on admin responses" + ); + + let json = response_json(response); + assert_eq!(json["ec_id"], ec_id.as_str()); + assert_eq!(json["store"], "test-store"); + assert_eq!(json["tombstone"], false); + assert_eq!( + json["entry"]["ids"]["bidstream.example"]["uid"], "uid-live", + "should echo the stored entry verbatim" + ); + + let eids = json["auction"]["eids"] + .as_array() + .expect("should have auction eids"); + assert_eq!(eids.len(), 1, "should resolve only the bidstream partner"); + assert_eq!(eids[0]["source"], "bidstream.example"); + assert_eq!(eids[0]["uids"][0]["id"], "uid-live"); + + let skipped = json["auction"]["skipped"] + .as_array() + .expect("should have skipped list"); + assert_eq!(skipped.len(), 2, "should report both filtered partners"); + assert!( + skipped + .iter() + .any(|s| s["source_domain"] == "disabled.example" + && s["reason"] == "bidstream_disabled"), + "should report the bidstream-disabled partner" + ); + assert!( + skipped.iter().any( + |s| s["source_domain"] == "unknown.example" && s["reason"] == "not_in_registry" + ), + "should report the unregistered partner" + ); + } + + #[test] + fn reports_tombstone_entries() { + let ec_id = test_ec_id(); + let kv = KvIdentityGraph::in_memory("test-store"); + kv.write_withdrawal_tombstone(&ec_id) + .expect("should write tombstone"); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert_eq!(json["tombstone"], true, "should flag tombstone entries"); + assert!( + json["auction"]["eids"] + .as_array() + .expect("should have auction eids") + .is_empty(), + "tombstone should resolve no EIDs" + ); + } + + #[test] + fn missing_entry_returns_404() { + let kv = KvIdentityGraph::in_memory("test-store"); + let req = get_request(&format!("/_ts/admin/ec/{}", test_ec_id())); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[test] + fn invalid_id_returns_400() { + let kv = KvIdentityGraph::in_memory("test-store"); + let req = get_request("/_ts/admin/ec/not-a-valid-id"); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[test] + fn corrupt_entry_returns_parse_error_and_raw_body() { + let ec_id = test_ec_id(); + let kv = kv_with_raw_body(&ec_id, "not json at all"); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!( + response.status(), + StatusCode::OK, + "corrupt entries should be inspectable, not opaque errors" + ); + let json = response_json(response); + assert!( + json["entry_error"] + .as_str() + .expect("should have entry_error") + .contains("failed to deserialize"), + "should describe the parse failure" + ); + assert_eq!(json["raw_body"], "not json at all"); + assert!(json.get("entry").is_none(), "should omit unparsed entry"); + assert!( + json.get("auction").is_none(), + "should omit auction view for unparseable entries" + ); + assert_eq!( + json["metadata"]["country"], "US", + "should still parse the stored metadata" + ); + } + + #[test] + fn invalid_schema_version_reports_validation_error() { + let ec_id = test_ec_id(); + let body = serde_json::json!({ + "v": 99, + "created": 1000, + "consent": { "ok": true, "updated": 1000 }, + "geo": { "country": "US" } + }) + .to_string(); + let kv = kv_with_raw_body(&ec_id, &body); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert!( + json["entry_error"] + .as_str() + .expect("should have entry_error") + .contains("failed validation"), + "should describe the validation failure" + ); + assert_eq!(json["entry"]["v"], 99, "should still show the parsed entry"); + assert!( + json.get("auction").is_none(), + "should omit auction view when the auction read would fail closed" + ); + } + + #[test] + fn bare_route_uses_ts_ec_cookie() { + let ec_id = test_ec_id(); + let kv = kv_with_entry(&ec_id, &sample_entry()); + let req = get_request_with_cookie("/_ts/admin/ec", &format!("other=1; ts-ec={ec_id}; x=2")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert_eq!( + json["ec_id"], + ec_id.as_str(), + "should resolve the EC ID from the ts-ec cookie" + ); + } + + #[test] + fn bare_route_without_cookie_returns_404() { + let kv = KvIdentityGraph::in_memory("test-store"); + let req = get_request("/_ts/admin/ec"); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let json = response_json(response); + assert!( + json["error"] + .as_str() + .expect("should have error message") + .contains("ts-ec cookie"), + "should explain the missing cookie" + ); + } + + #[test] + fn missing_identity_graph_returns_501() { + let req = get_request(&format!("/_ts/admin/ec/{}", test_ec_id())); + + let response = + handle_admin_ec_lookup(None, &test_registry(), &req).expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED); + } + + #[test] + fn kv_read_failure_propagates() { + let kv = KvIdentityGraph::failing("broken-store"); + let req = get_request(&format!("/_ts/admin/ec/{}", test_ec_id())); + + let result = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req); + + assert!(result.is_err(), "should propagate KV read failures"); + } +} diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 7be767557..3572581ce 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -21,7 +21,7 @@ use crate::error::TrustedServerError; use super::current_timestamp; use super::generation::ec_hash; -use super::kv_backend::{EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; +use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; use super::kv_types::{KvEntry, KvMetadata, KvNetwork}; use super::log_id; @@ -170,6 +170,25 @@ impl KvIdentityGraph { Ok((body, meta_str)) } + /// Reads the raw stored body, metadata, and generation for an EC ID key. + /// + /// Unlike [`Self::get`], the entry body is returned without + /// deserialization or validation, so corrupt or legacy-schema records can + /// still be inspected instead of failing closed. Used by the admin EC + /// lookup endpoint. + /// + /// Returns `Ok(None)` when the key does not exist. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store open or read failure. + pub fn lookup_raw( + &self, + ec_id: &str, + ) -> Result, Report> { + self.store.lookup(ec_id) + } + /// Reads the full entry and its generation marker for CAS writes. /// /// Returns `Ok(None)` when the key does not exist. diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 408ea9b32..50eda4d60 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -31,6 +31,7 @@ mod auth; +pub mod admin; pub mod batch_sync; pub mod consent; pub mod cookies; diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 03cc535c8..eb463acf3 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2200,9 +2200,18 @@ impl Settings { /// where any of these paths lack a matching handler, ensuring admin /// endpoints are always protected by authentication. /// Update [`ADMIN_ENDPOINTS`](Self::ADMIN_ENDPOINTS) when adding new - /// admin routes to `crates/trusted-server-adapter-fastly/src/main.rs`. - pub(crate) const ADMIN_ENDPOINTS: &[&str] = - &["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate"]; + /// admin routes to `crates/trusted-server-adapter-fastly/src/app.rs`. + /// + /// The `/_ts/admin/ec/{id}` entry is the literal router pattern; handler + /// path regexes are matched against it verbatim, so prefix-style admin + /// regexes (e.g. `^/_ts/admin`) cover it while regexes too narrow to + /// cover the parameterized route are rejected fail-closed. + pub(crate) const ADMIN_ENDPOINTS: &[&str] = &[ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/_ts/admin/ec", + "/_ts/admin/ec/{id}", + ]; /// Returns admin endpoint paths that no configured handler covers. /// @@ -5249,7 +5258,12 @@ origin_host_header_overide = "www.example.com""#, .expect("should check admin coverage"); assert_eq!( uncovered, - vec!["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate"], + vec![ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/_ts/admin/ec", + "/_ts/admin/ec/{id}", + ], "should report every admin endpoint as uncovered" ); } @@ -5283,7 +5297,11 @@ origin_host_header_overide = "www.example.com""#, .expect("should check admin coverage"); assert_eq!( uncovered, - vec!["/_ts/admin/keys/deactivate"], + vec![ + "/_ts/admin/keys/deactivate", + "/_ts/admin/ec", + "/_ts/admin/ec/{id}", + ], "should detect the admin endpoints not covered by the narrow handler" ); } From 12592bb0489371775f47bab74e186e56a6955a84 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 17 Jul 2026 18:46:43 +0530 Subject: [PATCH 145/395] Do not bot-gate the admin EC lookup KV graph The dispatch arm reused EcRequestState::kv_graph, which is deliberately None for clients that fail the browser gate. Operators hit this auth-gated endpoint with curl, so every lookup returned 501 as if no EC store were configured. Build the identity graph directly from settings instead, and document why the bot-gated copy must not be used. Also point the bare-route no-cookie 404 at the explicit-id route, since the ts-ec cookie (Domain-scoped, Secure) cannot exist on localhost. --- crates/trusted-server-adapter-fastly/src/app.rs | 6 +++++- crates/trusted-server-core/src/ec/admin.rs | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 71d906d52..b44b43703 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -569,8 +569,12 @@ async fn run_named_route( NamedRouteHandler::RotateKey => handle_rotate_key(&state.settings, services, req), NamedRouteHandler::DeactivateKey => handle_deactivate_key(&state.settings, services, req), NamedRouteHandler::AdminEcLookup => { + // Deliberately NOT `ec.kv_graph`: that copy is bot-gated (None for + // non-browser clients), and operators hit this auth-gated endpoint + // with curl. Build the graph directly from settings instead. + let kv = crate::maybe_identity_graph(&state.settings); let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; - handle_admin_ec_lookup(ec.kv_graph.as_ref(), &partner_registry, &req) + handle_admin_ec_lookup(kv.as_ref(), &partner_registry, &req) } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::BatchSync => { diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 54599d59d..6c256e44e 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -164,7 +164,8 @@ fn requested_ec_id(req: &Request) -> Result { return Err(Box::new(json_error( StatusCode::NOT_FOUND, - "no EC ID in path and no ts-ec cookie on the request", + "no EC ID in path and no ts-ec cookie on the request — pass \ + an explicit id: /_ts/admin/ec/{id}", ))); } } From 9869ac7024003bf6ef5686eba11b6d0a19a59a36 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 17 Jul 2026 19:58:44 +0530 Subject: [PATCH 146/395] Add admin endpoint to echo request EID cookies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds GET /_ts/admin/eids, complementing the EC lookup endpoint with the client-side half of EID propagation: it decodes the request's ts-eids and sharedId cookies and previews what cookie ingestion would write into the EC entry's ids map — matched partner UIDs (deduplicated exactly like the ingestion path) and unmatched sources that would be dropped. The endpoint always responds 200; missing or malformed cookies are reported in the payload rather than as errors. It is pure request inspection with no KV access, so every adapter serves the real handler. The path joins Settings::ADMIN_ENDPOINTS for basic-auth coverage validation. --- crates/trusted-server-adapter-axum/src/app.rs | 17 +- .../tests/routes.rs | 26 ++ .../src/app.rs | 11 + .../tests/routes.rs | 20 ++ .../trusted-server-adapter-fastly/src/app.rs | 29 +- crates/trusted-server-adapter-spin/src/app.rs | 19 +- .../tests/routes.rs | 19 ++ crates/trusted-server-core/src/ec/admin.rs | 259 +++++++++++++++++- .../trusted-server-core/src/ec/prebid_eids.rs | 6 +- crates/trusted-server-core/src/settings.rs | 3 + 10 files changed, 400 insertions(+), 9 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 12acbfc68..61f9e977d 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -12,6 +12,8 @@ use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; use trusted_server_core::proxy::{ @@ -253,6 +255,7 @@ enum NamedRouteHandler { VerifySignature, AdminNotSupported, AdminEcNotSupported, + AdminEidsLookup, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, @@ -280,7 +283,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_routes() -> [NamedRoute; 14] { +fn named_routes() -> [NamedRoute; 15] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -318,6 +321,13 @@ fn named_routes() -> [NamedRoute; 14] { primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEcNotSupported, }, + // Admin EIDs echo: pure request inspection (no KV), so the dev + // server serves the real handler. + NamedRoute { + path: "/_ts/admin/eids", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEidsLookup, + }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with // a 404, matching the Fastly and Cloudflare adapters: the production // basic-auth handler regex `^/_ts/admin` does not match them, and letting @@ -417,6 +427,11 @@ fn named_route_handler( ); Ok(resp) } + NamedRouteHandler::AdminEidsLookup => { + let partner_registry = + PartnerRegistry::from_config(&state.settings.ec.partners)?; + handle_admin_eids_lookup(&partner_registry, &req) + } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::Auction => { // Build the geo-aware EC context so the auction consent diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index 7c20e2dd2..f64c9361b 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -76,6 +76,7 @@ fn all_explicit_routes_are_registered() { ("POST", "/_ts/admin/keys/deactivate"), ("GET", "/_ts/admin/ec"), ("GET", "/_ts/admin/ec/{id}"), + ("GET", "/_ts/admin/eids"), ("POST", "/admin/keys/rotate"), ("POST", "/admin/keys/deactivate"), ("POST", "/auction"), @@ -290,6 +291,31 @@ async fn authenticated_admin_ec_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_eids_route_returns_200() { + // The EIDs echo is pure request inspection (no KV), so the dev server + // serves the real handler. + let mut svc = make_service(); + let req = Request::builder() + .method("GET") + .uri("/_ts/admin/eids") + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(AxumBody::empty()) + .expect("should build request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + assert_eq!( + resp.status().as_u16(), + 200, + "/_ts/admin/eids should serve the real EIDs echo handler" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { // Regression for the credential-leak finding: the production basic-auth regex diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 85eb09f1b..593522026 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -13,6 +13,8 @@ use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; #[cfg(target_arch = "wasm32")] use trusted_server_core::config_payload::settings_from_config_blob; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; use trusted_server_core::platform::RuntimeServices; @@ -486,6 +488,15 @@ fn build_router(state: &Arc) -> RouterService { .get("/_ts/admin/ec/{id}", |_ctx: RequestContext| async { Ok::(admin_ec_lookup_not_supported()) }) + // Admin EIDs echo: pure request inspection (no KV), so this + // adapter serves the real handler. + .get( + "/_ts/admin/eids", + make_handler(Arc::clone(&state), |s, _services, req| async move { + let partner_registry = PartnerRegistry::from_config(&s.settings.ec.partners)?; + handle_admin_eids_lookup(&partner_registry, &req) + }), + ) .post( "/auction", make_handler(Arc::clone(&state), |s, services, req| async move { diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index 7b048833b..8ecd020b7 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -217,6 +217,7 @@ fn all_explicit_routes_are_registered() { ("POST", "/_ts/admin/keys/deactivate"), ("GET", "/_ts/admin/ec"), ("GET", "/_ts/admin/ec/{id}"), + ("GET", "/_ts/admin/eids"), ("POST", "/auction"), ("GET", "/first-party/proxy"), ("GET", "/first-party/click"), @@ -292,6 +293,25 @@ async fn authenticated_admin_ec_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_eids_route_returns_200() { + // The EIDs echo is pure request inspection (no KV), so this adapter + // serves the real handler. + let req = request_builder() + .method("GET") + .uri("/_ts/admin/eids") + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 200, + "/_ts/admin/eids should serve the real EIDs echo handler" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_route_without_credentials_returns_401() { let router = test_router(); diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index b44b43703..1703b2d67 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -24,6 +24,7 @@ //! | POST | `/_ts/admin/keys/deactivate` | [`handle_deactivate_key`] | //! | GET | `/_ts/admin/ec` | [`handle_admin_ec_lookup`] | //! | GET | `/_ts/admin/ec/{id}` | [`handle_admin_ec_lookup`] | +//! | GET | `/_ts/admin/eids` | [`handle_admin_eids_lookup`] | //! | POST | `/_ts/api/v1/batch-sync` | [`handle_batch_sync`] | //! | GET | `/_ts/api/v1/identify` | [`handle_identify`] | //! | GET | `/_ts/set-tester` | [`handle_set_tester`] | @@ -100,7 +101,7 @@ use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::handle_admin_ec_lookup; +use trusted_server_core::ec::admin::{handle_admin_ec_lookup, handle_admin_eids_lookup}; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; @@ -576,6 +577,10 @@ async fn run_named_route( let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; handle_admin_ec_lookup(kv.as_ref(), &partner_registry, &req) } + NamedRouteHandler::AdminEidsLookup => { + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + handle_admin_eids_lookup(&partner_registry, &req) + } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::BatchSync => { // Dispatched by execute_named before EC state is built. @@ -999,6 +1004,7 @@ enum NamedRouteHandler { RotateKey, DeactivateKey, AdminEcLookup, + AdminEidsLookup, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, @@ -1063,6 +1069,13 @@ const NAMED_ROUTES: &[NamedRoute] = &[ primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEcLookup, }, + // Admin EIDs echo: decodes the request's ts-eids/sharedId cookies with + // an ingestion preview. Pure request inspection — no KV access. + NamedRoute { + path: "/_ts/admin/eids", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEidsLookup, + }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with a // 404 instead of executing key operations: the production basic-auth handler // regex `^/_ts/admin` does not match them, and letting them fall through to @@ -1670,6 +1683,20 @@ mod tests { "{path} must have GET as its only primary method" ); } + + let eids_route = NAMED_ROUTES + .iter() + .find(|route| route.path == "/_ts/admin/eids") + .expect("should register /_ts/admin/eids as a named route"); + assert!( + matches!(eids_route.handler, NamedRouteHandler::AdminEidsLookup), + "/_ts/admin/eids must map to the admin EIDs lookup handler" + ); + assert_eq!( + eids_route.primary_methods, + &[Method::GET], + "/_ts/admin/eids must have GET as its only primary method" + ); } #[test] diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 29ca574ff..51909998d 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -11,6 +11,8 @@ use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; @@ -141,7 +143,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 14] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 15] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), @@ -149,6 +151,7 @@ fn named_fallback_paths() -> [(&'static str, &'static [Method]); 14] { ("/_ts/admin/keys/deactivate", &[Method::POST]), ("/_ts/admin/ec", &[Method::GET]), ("/_ts/admin/ec/{id}", &[Method::GET]), + ("/_ts/admin/eids", &[Method::GET]), ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), ("/auction", &[Method::POST]), @@ -531,6 +534,19 @@ fn build_router(state: &Arc) -> RouterService { Ok::(admin_ec_lookup_not_supported()) }; + // Admin EIDs echo: pure request inspection (no KV), so this adapter + // serves the real handler. + let s = Arc::clone(&state); + let admin_eids_handler = move |ctx: RequestContext| { + let s = Arc::clone(&s); + async move { + let req = ctx.into_request(); + let result = PartnerRegistry::from_config(&s.settings.ec.partners) + .and_then(|registry| handle_admin_eids_lookup(®istry, &req)); + Ok::(result.unwrap_or_else(|e| http_error(&e))) + } + }; + // /auction let s = Arc::clone(&state); let auction_handler = move |ctx: RequestContext| { @@ -757,6 +773,7 @@ fn build_router(state: &Arc) -> RouterService { // adapter has no store to read. .get("/_ts/admin/ec", admin_ec_not_supported_handler) .get("/_ts/admin/ec/{id}", admin_ec_not_supported_handler) + .get("/_ts/admin/eids", admin_eids_handler) .post("/auction", auction_handler) .get("/__ts/page-bids", page_bids_handler) .route( diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 4194baea4..d502a3944 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -139,6 +139,25 @@ async fn authenticated_admin_ec_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_eids_route_returns_200() { + // The EIDs echo is pure request inspection (no KV), so this adapter + // serves the real handler. + let req = request_builder() + .method("GET") + .uri("/_ts/admin/eids") + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 200, + "/_ts/admin/eids should serve the real EIDs echo handler" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn health_route_returns_ok() { // Parity with the Fastly/Axum adapters: GET /health is a cheap liveness probe diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 6c256e44e..0724d1f4b 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -1,4 +1,4 @@ -//! Admin endpoint for inspecting EC identity graph entries. +//! Admin endpoints for inspecting EC identity state. //! //! Serves `GET /_ts/admin/ec` (EC ID taken from the request's `ts-ec` //! cookie) and `GET /_ts/admin/ec/{id}` (explicit EC ID). Returns the raw @@ -6,9 +6,13 @@ //! attach, so operators can debug KV-to-auction propagation without KV //! console access. //! +//! Also serves `GET /_ts/admin/eids`, which echoes the request's `ts-eids` +//! and `sharedId` cookies with an ingestion preview — the client-side half +//! of EID propagation that is never stored server-side. +//! //! Authentication is enforced by the `^/_ts/admin` basic-auth handler //! configuration; startup validation rejects configs that leave these paths -//! uncovered (see `Settings::ADMIN_ENDPOINTS`). Because the endpoint is +//! uncovered (see `Settings::ADMIN_ENDPOINTS`). Because the endpoints are //! auth-gated and operator-facing, responses intentionally include full //! internal detail (raw consent strings, partner UIDs, parse errors). @@ -19,7 +23,7 @@ use serde_json::Value as JsonValue; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt as _}; -use crate::constants::COOKIE_TS_EC; +use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EC, COOKIE_TS_EIDS}; use crate::error::TrustedServerError; use crate::openrtb::Eid; @@ -29,6 +33,10 @@ use super::kv::KvIdentityGraph; use super::kv_backend::EcKvLookup; use super::kv_types::{KvEntry, KvMetadata}; use super::log_id; +use super::prebid_eids::{ + collect_prebid_eid_updates, collect_sharedid_update, dedupe_partner_updates, + parse_prebid_eids_cookie, +}; use super::registry::PartnerRegistry; /// Route prefix shared by the cookie-based and explicit-ID lookup routes. @@ -271,6 +279,125 @@ fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEid AuctionEidsView { eids, skipped } } +/// Admin EIDs echo payload. +#[derive(Debug, Serialize)] +struct AdminEidsResponse { + /// Whether a `ts-eids` cookie was present on the request. + cookie_present: bool, + /// EIDs parsed from the `ts-eids` cookie. Absent when the cookie is + /// missing or failed to parse. + #[serde(skip_serializing_if = "Option::is_none")] + eids: Option>, + /// Parse failure detail when the `ts-eids` cookie could not be decoded. + #[serde(skip_serializing_if = "Option::is_none")] + parse_error: Option, + /// Whether a `sharedId` cookie was present on the request. + sharedid_present: bool, + /// Number of partners configured in the registry. + partners_configured: usize, + /// Preview of what cookie ingestion would write into the EC entry's + /// `ids` map on a navigation carrying these cookies. + ingest: IngestPreview, +} + +/// What cookie ingestion would store, and what it would drop. +#[derive(Debug, Serialize)] +struct IngestPreview { + /// Cookie sources matched to a configured partner, with the UID that + /// would be stored (deduplicated exactly like the ingestion path). + matched: Vec, + /// `ts-eids` sources with no configured partner; dropped on ingestion. + unmatched: Vec, +} + +/// A cookie-derived partner UID that ingestion would store. +#[derive(Debug, Serialize)] +struct MatchedPartnerId { + /// Partner namespace key in the EC entry's `ids` map. + source_domain: String, + /// The UID that would be stored. + uid: String, +} + +/// Handles `GET /_ts/admin/eids`. +/// +/// Echoes the request's `ts-eids` and `sharedId` cookies: the parsed EID +/// list plus a preview of what cookie ingestion would write into the EC +/// entry's `ids` map given the configured partner registry. Pure request +/// inspection — no KV access — so it works on every adapter. +/// +/// Always responds `200 OK`; missing or malformed cookies are reported in +/// the payload instead of as errors. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] only when the response +/// payload fails JSON serialization. +pub fn handle_admin_eids_lookup( + registry: &PartnerRegistry, + req: &Request, +) -> Result, Report> { + let eids_cookie = extract_cookie_value(req, COOKIE_TS_EIDS); + let sharedid_cookie = extract_cookie_value(req, COOKIE_SHAREDID); + + let (eids, parse_error) = match &eids_cookie { + None => (None, None), + Some(value) => match parse_prebid_eids_cookie(value) { + Ok(parsed) => (Some(parsed), None), + Err(error) => ( + None, + Some(format!("failed to parse ts-eids cookie: {error}")), + ), + }, + }; + + // Mirror the ingestion path (`ingest_eid_cookies`): collect matches from + // both cookies, then dedupe the same way so the preview reports exactly + // what a navigation would store. + let mut updates = Vec::new(); + if let Some(value) = &eids_cookie { + updates.extend(collect_prebid_eid_updates(value, registry)); + } + if let Some(value) = &sharedid_cookie + && let Some(update) = collect_sharedid_update(value, registry) + { + updates.push(update); + } + let matched = dedupe_partner_updates(updates) + .into_iter() + .map(|update| MatchedPartnerId { + source_domain: update.partner_id, + uid: update.uid, + }) + .collect(); + + let unmatched = eids + .as_ref() + .map(|parsed| { + parsed + .iter() + .filter(|eid| registry.find_by_source_domain(&eid.source).is_none()) + .map(|eid| eid.source.clone()) + .collect() + }) + .unwrap_or_default(); + + let payload = AdminEidsResponse { + cookie_present: eids_cookie.is_some(), + eids, + parse_error, + sharedid_present: sharedid_cookie.is_some(), + partners_configured: registry.len(), + ingest: IngestPreview { matched, unmatched }, + }; + + let body = + serde_json::to_string(&payload).change_context(TrustedServerError::Configuration { + message: "failed to serialize admin EIDs response".to_owned(), + })?; + Ok(json_response(StatusCode::OK, body)) +} + fn extract_cookie_value(req: &Request, name: &str) -> Option { let cookie_header = req .headers() @@ -305,6 +432,8 @@ fn json_response(status: StatusCode, body: String) -> Response { mod tests { use std::time::Duration; + use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; + use super::*; use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_backend::{EcKvStore as _, EcKvWrite, EcKvWriteMode}; @@ -624,4 +753,128 @@ mod tests { assert!(result.is_err(), "should propagate KV read failures"); } + + fn eids_cookie_for(entries: &serde_json::Value) -> String { + BASE64.encode(entries.to_string()) + } + + #[test] + fn eids_lookup_without_cookies_returns_empty_payload() { + let req = get_request("/_ts/admin/eids"); + + let response = + handle_admin_eids_lookup(&test_registry(), &req).expect("should handle eids lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert_eq!(json["cookie_present"], false); + assert_eq!(json["sharedid_present"], false); + assert_eq!(json["partners_configured"], 2); + assert!( + json["ingest"]["matched"] + .as_array() + .expect("should have matched list") + .is_empty(), + "should preview no matches without cookies" + ); + } + + #[test] + fn eids_lookup_parses_cookie_and_previews_ingestion() { + let cookie = eids_cookie_for(&serde_json::json!([ + { + "source": "bidstream.example", + "uids": [{ "id": "uid-configured", "atype": 1 }] + }, + { + "source": "unknown.example", + "uids": [{ "id": "uid-unknown", "atype": 1 }] + } + ])); + let req = get_request_with_cookie("/_ts/admin/eids", &format!("ts-eids={cookie}")); + + let response = + handle_admin_eids_lookup(&test_registry(), &req).expect("should handle eids lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert_eq!(json["cookie_present"], true); + assert_eq!( + json["eids"] + .as_array() + .expect("should have parsed eids") + .len(), + 2, + "should echo both parsed EID sources" + ); + + let matched = json["ingest"]["matched"] + .as_array() + .expect("should have matched list"); + assert_eq!(matched.len(), 1, "should match only the configured partner"); + assert_eq!(matched[0]["source_domain"], "bidstream.example"); + assert_eq!(matched[0]["uid"], "uid-configured"); + + let unmatched = json["ingest"]["unmatched"] + .as_array() + .expect("should have unmatched list"); + assert_eq!(unmatched.len(), 1, "should report the unregistered source"); + assert_eq!(unmatched[0], "unknown.example"); + } + + #[test] + fn eids_lookup_reports_parse_error() { + let req = get_request_with_cookie("/_ts/admin/eids", "ts-eids=!!!not-base64!!!"); + + let response = + handle_admin_eids_lookup(&test_registry(), &req).expect("should handle eids lookup"); + + assert_eq!( + response.status(), + StatusCode::OK, + "malformed cookies should be reported, not errored" + ); + let json = response_json(response); + assert_eq!(json["cookie_present"], true); + assert!( + json["parse_error"] + .as_str() + .expect("should have parse_error") + .contains("ts-eids"), + "should describe the parse failure" + ); + assert!(json.get("eids").is_none(), "should omit unparsed eids"); + assert!( + json["ingest"]["matched"] + .as_array() + .expect("should have matched list") + .is_empty(), + "unparseable cookie should preview no matches" + ); + } + + #[test] + fn eids_lookup_includes_sharedid_match() { + let registry = PartnerRegistry::from_config(&[ + make_test_partner("bidstream.example", true), + make_test_partner("sharedid.org", true), + ]) + .expect("should build sharedid test registry"); + let req = get_request_with_cookie("/_ts/admin/eids", "sharedId=shared-uid-123"); + + let response = + handle_admin_eids_lookup(®istry, &req).expect("should handle eids lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert_eq!(json["cookie_present"], false); + assert_eq!(json["sharedid_present"], true); + + let matched = json["ingest"]["matched"] + .as_array() + .expect("should have matched list"); + assert_eq!(matched.len(), 1, "should match the sharedid partner"); + assert_eq!(matched[0]["source_domain"], "sharedid.org"); + assert_eq!(matched[0]["uid"], "shared-uid-123"); + } } diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 9f22b78e2..5003304dd 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -179,7 +179,7 @@ fn ingest_eid_cookies_with_writer( } } -fn collect_prebid_eid_updates( +pub(crate) fn collect_prebid_eid_updates( cookie_value: &str, registry: &PartnerRegistry, ) -> Vec { @@ -213,7 +213,7 @@ fn collect_prebid_eid_updates( updates } -fn dedupe_partner_updates(updates: Vec) -> Vec { +pub(crate) fn dedupe_partner_updates(updates: Vec) -> Vec { let mut latest = std::collections::BTreeMap::new(); for update in updates { latest.insert(update.partner_id, update.uid); @@ -250,7 +250,7 @@ pub fn ingest_sharedid_cookie( ingest_eid_cookies(None, Some(cookie_value), ec_id, kv, registry); } -fn collect_sharedid_update( +pub(crate) fn collect_sharedid_update( cookie_value: &str, registry: &PartnerRegistry, ) -> Option { diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index eb463acf3..1a291479e 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2211,6 +2211,7 @@ impl Settings { "/_ts/admin/keys/deactivate", "/_ts/admin/ec", "/_ts/admin/ec/{id}", + "/_ts/admin/eids", ]; /// Returns admin endpoint paths that no configured handler covers. @@ -5263,6 +5264,7 @@ origin_host_header_overide = "www.example.com""#, "/_ts/admin/keys/deactivate", "/_ts/admin/ec", "/_ts/admin/ec/{id}", + "/_ts/admin/eids", ], "should report every admin endpoint as uncovered" ); @@ -5301,6 +5303,7 @@ origin_host_header_overide = "www.example.com""#, "/_ts/admin/keys/deactivate", "/_ts/admin/ec", "/_ts/admin/ec/{id}", + "/_ts/admin/eids", ], "should detect the admin endpoints not covered by the narrow handler" ); From 8dc31cc4e4eff0a179c29368f4744e1ca7ea9636 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 18 Jul 2026 10:01:15 +0530 Subject: [PATCH 147/395] Add ISO 8601 companions to admin EC lookup timestamps Review feedback on the admin EC lookup asked for readable dates. The echoed entry now carries derived created_iso and consent.updated_iso fields (yyyy-MM-ddTHH:mm:ss.SSSZ) next to the stored unix-seconds values, which stay untouched so the echo remains faithful to KV. --- crates/trusted-server-core/src/ec/admin.rs | 49 +++++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 0724d1f4b..a4e6465ab 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -55,7 +55,9 @@ struct AdminEcLookupResponse { /// (`consent.ok = false`). Absent when the body failed to parse. #[serde(skip_serializing_if = "Option::is_none")] tombstone: Option, - /// The stored entry, re-serialized verbatim. Absent when the body + /// The stored entry, re-serialized verbatim except for derived + /// `created_iso` / `updated_iso` companions added next to the stored + /// unix-seconds timestamps for readability. Absent when the body /// failed to deserialize (see `entry_error` / `raw_body`). #[serde(skip_serializing_if = "Option::is_none")] entry: Option, @@ -226,7 +228,7 @@ fn build_lookup_response( )); } } - payload.entry = Some(serde_json::to_value(&entry).expect("should serialize KvEntry")); + payload.entry = Some(entry_json_with_iso_timestamps(&entry)); } Err(error) => { payload.entry_error = Some(format!("failed to deserialize entry: {error}")); @@ -253,6 +255,37 @@ fn build_lookup_response( payload } +/// Serializes an entry, adding derived ISO 8601 companions next to the +/// stored unix-seconds timestamps (`created_iso`, `consent.updated_iso`). +/// +/// The stored numeric values stay untouched so the echo remains faithful to +/// what is in KV; the ISO fields exist purely for operator readability. +fn entry_json_with_iso_timestamps(entry: &KvEntry) -> JsonValue { + let mut entry_json = serde_json::to_value(entry).expect("should serialize KvEntry"); + + if let Some(object) = entry_json.as_object_mut() { + if let Some(iso) = iso_timestamp(entry.created) { + object.insert("created_iso".to_owned(), JsonValue::String(iso)); + } + if let Some(consent) = object.get_mut("consent").and_then(JsonValue::as_object_mut) + && let Some(iso) = iso_timestamp(entry.consent.updated) + { + consent.insert("updated_iso".to_owned(), JsonValue::String(iso)); + } + } + + entry_json +} + +/// Formats a unix-seconds timestamp as ISO 8601 (`yyyy-MM-ddTHH:mm:ss.SSSZ`). +/// +/// Returns `None` for values outside the representable date range. +fn iso_timestamp(unix_seconds: u64) -> Option { + let unix_seconds = i64::try_from(unix_seconds).ok()?; + chrono::DateTime::from_timestamp(unix_seconds, 0) + .map(|datetime| datetime.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) +} + /// Derives the auction EID view for a valid entry, mirroring the filters in /// [`resolve_partner_ids`] and reporting why each stored ID was skipped. fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEidsView { @@ -559,6 +592,18 @@ mod tests { json["entry"]["ids"]["bidstream.example"]["uid"], "uid-live", "should echo the stored entry verbatim" ); + assert_eq!( + json["entry"]["created"], 1_741_824_000_u64, + "should keep the stored unix-seconds timestamp" + ); + assert_eq!( + json["entry"]["created_iso"], "2025-03-13T00:00:00.000Z", + "should add an ISO 8601 companion for created" + ); + assert_eq!( + json["entry"]["consent"]["updated_iso"], "2025-03-13T00:00:00.000Z", + "should add an ISO 8601 companion for consent.updated" + ); let eids = json["auction"]["eids"] .as_array() From bc13f2773ba97e8de07f7ba6ef6f3d0a3e37427f Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:38:03 -0700 Subject: [PATCH 148/395] Upgrade EdgeZero to the deploy-actions branch Point the edgezero-* dependencies at the feature/edgezero-deploy-actions branch (PR #316) and adapt Trusted Server to its API changes: - Wire the new ts CLI subcommands surfaced by edgezero-cli: active-version, healthcheck, and rollback, plus deploy --stage and a --version flag, with argument-parsing coverage. - Migrate TrustedServerAppConfig to the AppConfigMeta::secret_fields() method that replaces the removed SECRET_FIELDS associated constant. --- Cargo.lock | 37 +++-- Cargo.toml | 12 +- crates/trusted-server-cli/src/run.rs | 174 ++++++++++++++++++++++- crates/trusted-server-core/src/config.rs | 4 +- 4 files changed, 205 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68f14e753..bd92ec5ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1398,7 +1398,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "toml", ] @@ -1406,7 +1406,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-trait", @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-trait", @@ -1449,7 +1449,7 @@ dependencies = [ "log", "serde_json", "tempfile", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", "worker", ] @@ -1457,7 +1457,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-stream", @@ -1479,14 +1479,14 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.18", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-trait", @@ -1506,14 +1506,14 @@ dependencies = [ "subtle", "thiserror 2.0.18", "toml", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "chrono", "clap", @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-compression", @@ -1569,7 +1569,7 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "log", "proc-macro2", @@ -5074,6 +5074,19 @@ dependencies = [ "winnow 0.7.15", ] +[[package]] +name = "toml_edit" +version = "0.25.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.3", +] + [[package]] name = "toml_parser" version = "1.1.2+spec-1.1.0" @@ -5339,7 +5352,7 @@ dependencies = [ "tokio", "tokio-rustls", "toml", - "toml_edit", + "toml_edit 0.23.10+spec-1.0.0", "trusted-server-core", "url", "webpki-roots", diff --git a/Cargo.toml b/Cargo.toml index 695099d49..54569599b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,12 +53,12 @@ criterion = { version = "0.5", default-features = false, features = ["cargo_benc derive_more = { version = "2.0", features = ["display", "error"] } directories = "5" ed25519-dalek = { version = "2.2", features = ["rand_core"] } -edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-cli = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4" } -edgezero-core = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } +edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-cli = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions" } +edgezero-core = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } env_logger = "0.11" error-stack = "0.6" fastly = "0.12" diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 1b0bdfa29..b3be949a5 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -2,8 +2,8 @@ use std::process; use clap::{Parser, Subcommand}; use edgezero_cli::args::{ - AuthArgs, BuildArgs, ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, DeployArgs, - ProvisionArgs, ServeArgs, + ActiveVersionArgs, AuthArgs, BuildArgs, ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, + DeployArgs, HealthcheckArgs, ProvisionArgs, RollbackArgs, ServeArgs, }; use trusted_server_core::config::TrustedServerAppConfig; @@ -13,7 +13,7 @@ use crate::commands::config::init::{ConfigInitArgs, run_config_init}; use crate::prebid_bundle::{NpmPrebidBundleGenerator, PrebidBundleArgs, run_bundle}; #[derive(Debug, Parser)] -#[command(name = "ts", about = "Trusted Server CLI")] +#[command(name = "ts", version, about = "Trusted Server CLI")] struct Args { #[command(subcommand)] command: Command, @@ -21,6 +21,8 @@ struct Args { #[derive(Debug, Subcommand)] enum Command { + /// Print the currently active deployment version for a target adapter. + ActiveVersion(ActiveVersionArgs), /// Audit a public page and write draft Trusted Server artifacts. Audit(AuditArgs), /// Sign in / out / status against an `EdgeZero` adapter. @@ -32,10 +34,14 @@ enum Command { Config(ConfigCommand), /// Deploy the project through a target adapter. Deploy(DeployArgs), + /// Probe a deployed version until it reports healthy. + Healthcheck(HealthcheckArgs), /// Trusted Server Prebid commands. Prebid(PrebidArgs), /// Provision platform resources through a target adapter. Provision(ProvisionArgs), + /// Roll a service back to a previously active deployment version. + Rollback(RollbackArgs), /// Serve the project locally through a target adapter. Serve(ServeArgs), /// Local developer tools (e.g. the macOS-only production-hostname proxy). @@ -79,6 +85,7 @@ pub fn run_from_env() -> Result<(), String> { fn dispatch(args: Args) -> Result<(), String> { match args.command { + Command::ActiveVersion(args) => edgezero_cli::run_active_version(&args), Command::Audit(args) => { let stdout = std::io::stdout(); let mut out = stdout.lock(); @@ -102,6 +109,7 @@ fn dispatch(args: Args) -> Result<(), String> { edgezero_cli::run_config_validate_typed::(&args) } Command::Deploy(args) => edgezero_cli::run_deploy(&args), + Command::Healthcheck(args) => edgezero_cli::run_healthcheck(&args), Command::Prebid(prebid) => { let mut generator = NpmPrebidBundleGenerator; let mut stdout = std::io::stdout(); @@ -113,6 +121,7 @@ fn dispatch(args: Args) -> Result<(), String> { } } Command::Provision(args) => edgezero_cli::run_provision(&args), + Command::Rollback(args) => edgezero_cli::run_rollback(&args), Command::Serve(args) => edgezero_cli::run_serve(&args), Command::Dev(command) => crate::commands::dev::run(command), } @@ -131,6 +140,165 @@ mod tests { Args::try_parse_from(args).expect("should parse args") } + #[test] + fn parses_active_version() { + let args = parse(&[ + "ts", + "active-version", + "--adapter", + "fastly", + "--service-id", + "service-123", + ]); + let Command::ActiveVersion(active_version) = args.command else { + panic!("expected active-version command"); + }; + assert_eq!(active_version.adapter, "fastly"); + assert_eq!(active_version.service_id, "service-123"); + } + + #[test] + fn parses_healthcheck_with_retry_defaults() { + let args = parse(&[ + "ts", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + "--domain", + "edge.example", + ]); + let Command::Healthcheck(healthcheck) = args.command else { + panic!("expected healthcheck command"); + }; + assert_eq!(healthcheck.domain, "edge.example"); + assert_eq!(healthcheck.version, "7"); + assert_eq!(healthcheck.retry, 3, "should default to 3 retries"); + assert_eq!( + healthcheck.retry_delay, 5, + "should default to a 5s retry delay" + ); + assert_eq!(healthcheck.timeout, 10, "should default to a 10s timeout"); + assert!(!healthcheck.staging, "should probe production by default"); + } + + #[test] + fn parses_healthcheck_with_staging_overrides() { + let args = parse(&[ + "ts", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + "--domain", + "edge.example", + "--staging", + "--retry", + "9", + "--retry-delay", + "2", + "--timeout", + "30", + ]); + let Command::Healthcheck(healthcheck) = args.command else { + panic!("expected healthcheck command"); + }; + assert!(healthcheck.staging); + assert_eq!(healthcheck.retry, 9); + assert_eq!(healthcheck.retry_delay, 2); + assert_eq!(healthcheck.timeout, 30); + } + + #[test] + fn healthcheck_requires_domain() { + Args::try_parse_from([ + "ts", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + ]) + .expect_err("should reject healthcheck without a domain"); + } + + #[test] + fn parses_rollback_with_explicit_target() { + let args = parse(&[ + "ts", + "rollback", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "8", + "--rollback-to", + "7", + ]); + let Command::Rollback(rollback) = args.command else { + panic!("expected rollback command"); + }; + assert_eq!(rollback.version, "8"); + assert_eq!(rollback.rollback_to, Some("7".to_owned())); + assert!(!rollback.staging); + } + + #[test] + fn parses_staging_rollback_without_target() { + let args = parse(&[ + "ts", + "rollback", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "8", + "--staging", + ]); + let Command::Rollback(rollback) = args.command else { + panic!("expected rollback command"); + }; + assert!(rollback.staging); + assert_eq!( + rollback.rollback_to, None, + "staging rollback should not need an explicit target" + ); + } + + #[test] + fn rollback_requires_service_id() { + Args::try_parse_from(["ts", "rollback", "--adapter", "fastly", "--version", "8"]) + .expect_err("should reject rollback without a service id"); + } + + #[test] + fn parses_deploy_with_staging_flags() { + let args = parse(&[ + "ts", + "deploy", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--stage", + ]); + let Command::Deploy(deploy) = args.command else { + panic!("expected deploy command"); + }; + assert_eq!(deploy.service_id, Some("service-123".to_owned())); + assert!(deploy.stage); + } + #[test] fn parses_audit_with_default_outputs() { let args = parse(&["ts", "audit", "https://publisher.example"]); diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 7bbecd747..991ed7a2f 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -110,7 +110,9 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { // app-config blob. Migrating app-level secrets to `EdgeZero` secret-store // references needs nested/array extraction support and operator migration // work tracked separately. - const SECRET_FIELDS: &'static [edgezero_core::app_config::SecretField] = &[]; + fn secret_fields() -> Vec { + Vec::new() + } } /// Runs Trusted Server deploy-time validation for pushed app config. From 5cc00c71a632f8d4ec25efc8246908cee060e2b1 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 21 Jul 2026 10:46:52 +0530 Subject: [PATCH 149/395] Add SSAT debug comment configuration spec and implementation plan --- .../2026-07-20-ssat-debug-comment-config.md | 699 ++++++++++++++++++ ...-07-20-ssat-debug-comment-config-design.md | 397 ++++++++++ 2 files changed, 1096 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-20-ssat-debug-comment-config.md create mode 100644 docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md diff --git a/docs/superpowers/plans/2026-07-20-ssat-debug-comment-config.md b/docs/superpowers/plans/2026-07-20-ssat-debug-comment-config.md new file mode 100644 index 000000000..ed7d07a6f --- /dev/null +++ b/docs/superpowers/plans/2026-07-20-ssat-debug-comment-config.md @@ -0,0 +1,699 @@ +# SSAT Debug Comment Configuration Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the SSAT `` auction dump configurable — section toggles, a metadata-key subset, and an opt-in `Full` verbosity that surfaces raw per-bidder request/response data — while keeping the existing fail-closed redaction unconditional. + +**Architecture:** One new config struct (`AuctionDebugCommentOptions`) and one new enum (`AuctionDebugCommentVerbosity`) in `settings.rs`, threaded as a parameter through the three existing render functions in `publisher.rs`. No new files, no new crates. + +**Tech Stack:** Rust, serde, existing `trusted-server-core` auction/settings modules. + +**Spec:** `docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md` — read it first for the full rationale (security invariants, non-goals, edge cases). This plan implements it; it doesn't re-derive it. + +--- + +## File Structure + +| File | Responsibility | +|---|---| +| `crates/trusted-server-core/src/settings.rs` | `AuctionDebugCommentOptions`, `AuctionDebugCommentVerbosity`, `AUCTION_DEBUG_METADATA_ALLOWLIST`, wiring into `DebugConfig` and `finalize_deserialized` | +| `crates/trusted-server-core/src/publisher.rs` | `redact_response_for_dump`, `redact_bid_for_dump`, `prepend_auction_debug_comment` — all three gain an `options` parameter; production + test call sites updated; new tests | +| `trusted-server.example.toml` | Document the new `[debug.auction_html_comment_options]` table | + +No file split needed — both touched files stay well under the codebase's existing size (settings.rs and publisher.rs are already large multi-struct files; this adds one cohesive struct + enum to each, following the file's existing pattern of many sibling config structs). + +--- + +### Task 1: Config struct in settings.rs + +**Files:** +- Modify: `crates/trusted-server-core/src/settings.rs:1894-1924` (the `DebugConfig` block) +- Modify: `crates/trusted-server-core/src/settings.rs:2038-2059` (`finalize_deserialized`) +- Test: same file, `#[cfg(test)] mod tests` block (search for an existing `mod tests` near the bottom of settings.rs to append into) + +- [ ] **Step 1: Write the failing test for `AuctionDebugCommentOptions::default()`** + +Add to the settings.rs test module: + +```rust +#[test] +fn auction_debug_comment_options_default_matches_serde_defaults() { + let opts = AuctionDebugCommentOptions::default(); + assert!(opts.include_provider_responses, "should default to true"); + assert!(opts.include_mediator_response, "should default to true"); + assert!(opts.include_bids, "should default to true"); + assert_eq!( + opts.metadata_keys, + AUCTION_DEBUG_METADATA_ALLOWLIST + .iter() + .map(|s| s.to_string()) + .collect::>(), + "should default metadata_keys to the full allowlist" + ); + assert_eq!( + opts.verbosity, + AuctionDebugCommentVerbosity::Redacted, + "should default to Redacted" + ); +} + +#[test] +fn auction_debug_comment_options_normalize_trims_and_drops_empty_keys() { + let mut opts = AuctionDebugCommentOptions { + metadata_keys: vec![" status ".to_string(), "".to_string(), "warnings".to_string()], + ..AuctionDebugCommentOptions::default() + }; + opts.normalize(); + assert_eq!(opts.metadata_keys, vec!["status".to_string(), "warnings".to_string()]); +} + +#[test] +fn bad_verbosity_string_fails_config_load() { + // Deserialize AuctionDebugCommentOptions directly, not a full Settings — + // Settings has required fields with no #[serde(default)] (e.g. + // `publisher`), so a full-Settings fixture missing them would fail with + // "missing field `publisher`" regardless of whether `verbosity` itself + // deserialized correctly, testing the wrong thing. + let result: Result = + toml::from_str(r#"verbosity = "everything""#); + assert!(result.is_err(), "unrecognized verbosity must fail to deserialize, not silently fall back"); +} +``` + +Run: `cargo test -p trusted-server-core auction_debug_comment_options -- --nocapture` +Expected: FAIL to compile — `AuctionDebugCommentOptions`, `AuctionDebugCommentVerbosity`, `AUCTION_DEBUG_METADATA_ALLOWLIST` don't exist yet. + +(Use `cargo test -p trusted-server-core`, not `cargo test-axum` — the latter is an alias for `cargo test -p trusted-server-adapter-axum` only per `.cargo/config.toml` and will NOT build or run `trusted-server-core`'s own `#[cfg(test)]` modules, silently reporting "0 passed; 0 failed" instead of actually exercising these tests. This applies to every test command in Task 1 and Task 3 below.) + +- [ ] **Step 2: Implement the struct, enum, and allowlist constant** + +Insert into `settings.rs`, near the existing `DebugConfig` (around line 1894), replacing the old bool-only struct: + +```rust +/// Metadata keys safe to surface in the `ts-debug` auction comment. +/// +/// Fail-closed superset: any key not listed here — notably `debug`, which +/// carries the resolved OpenRTB request (EC ID, `user.ext.eids`, the TC +/// consent string, `device.ip`, `device.geo`) plus per-bidder `httpcalls` — +/// is dropped in [`AuctionDebugCommentVerbosity::Redacted`] mode regardless +/// of what an operator lists in +/// [`AuctionDebugCommentOptions::metadata_keys`]. `metadata_keys` is a subset +/// selector against this const, never a way to add new keys. +pub(crate) const AUCTION_DEBUG_METADATA_ALLOWLIST: &[&str] = &[ + "error_type", + "http_status", + "message", + "upstream_message", + "upstream_message_truncated", + "responsetimemillis", + "errors", + "warnings", + "bidstatus", +]; + +fn default_true() -> bool { + true +} + +fn default_auction_debug_metadata_keys() -> Vec { + AUCTION_DEBUG_METADATA_ALLOWLIST + .iter() + .map(|s| s.to_string()) + .collect() +} + +/// Behavior of the `` auction dump. Only consulted when +/// [`DebugConfig::auction_html_comment`] is true. +/// +/// `deny_unknown_fields` matches the convention used by 17 of the ~19 +/// sibling config structs in this file, including the `DebugConfig` this +/// struct nests under: an operator typo (e.g. `metadata_key` instead of +/// `metadata_keys`) must fail config load loudly, not be silently ignored. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AuctionDebugCommentOptions { + /// Include the `provider_responses` section at all. + #[serde(default = "default_true")] + pub include_provider_responses: bool, + + /// Include `mediator_response` when a mediator ran. + #[serde(default = "default_true")] + pub include_mediator_response: bool, + + /// Include each provider's `bids` array (vs. status/metadata only). + #[serde(default = "default_true")] + pub include_bids: bool, + + /// Subset of [`AUCTION_DEBUG_METADATA_ALLOWLIST`] to surface in + /// [`AuctionDebugCommentVerbosity::Redacted`] mode. Keys outside the + /// fixed allowlist are always dropped, config or not. Ignored when + /// `verbosity` is `Full`. + #[serde(default = "default_auction_debug_metadata_keys")] + pub metadata_keys: Vec, + + /// `Redacted` (default): `metadata_keys` subset only, creative preview + /// truncated to `MAX_BID_CREATIVE_DUMP_BYTES`. + /// `Full`: raw `response.metadata` verbatim, including the `debug` + /// subtree (httpcalls/resolvedrequest) when present, and no creative + /// truncation. The total dump byte cap and comment-terminator + /// neutralization still apply unconditionally. + /// + /// NEVER enable `Full` in production — identity-bearing request/response + /// data becomes visible to any visitor via view-source. + #[serde(default)] + pub verbosity: AuctionDebugCommentVerbosity, +} + +impl Default for AuctionDebugCommentOptions { + fn default() -> Self { + Self { + include_provider_responses: true, + include_mediator_response: true, + include_bids: true, + metadata_keys: default_auction_debug_metadata_keys(), + verbosity: AuctionDebugCommentVerbosity::Redacted, + } + } +} + +impl AuctionDebugCommentOptions { + pub(crate) fn normalize(&mut self) { + self.metadata_keys = self + .metadata_keys + .drain(..) + .map(|key| key.trim().to_string()) + .filter(|key| !key.is_empty()) + .collect(); + } +} + +/// Verbosity of the `ts-debug` auction comment. See +/// [`AuctionDebugCommentOptions::verbosity`]. +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuctionDebugCommentVerbosity { + #[default] + Redacted, + Full, +} +``` + +Then update `DebugConfig` itself (replacing the doc comment on `auction_html_comment` isn't needed — it's unchanged — just add the new field after `auction_html_comment`): + +```rust + #[serde(default)] + pub auction_html_comment: bool, + + /// Content and verbosity of the `auction_html_comment` dump. Ignored + /// when `auction_html_comment` is false. + #[serde(default)] + pub auction_html_comment_options: AuctionDebugCommentOptions, +``` + +- [ ] **Step 3: Wire normalize() into finalize_deserialized** + +In `settings.rs:2038-2059`, add one line after the existing normalize calls: + +```rust + pub(crate) fn finalize_deserialized( + mut settings: Self, + validation_label: &str, + ) -> Result> { + settings.integrations.normalize(); + settings.proxy.normalize(); + settings.image_optimizer.normalize(); + settings.debug.auction_html_comment_options.normalize(); + settings.consent.validate(); +``` + +- [ ] **Step 4: Run tests, verify pass** + +Run: `cargo test -p trusted-server-core auction_debug_comment_options -- --nocapture` +Expected: PASS (all 3 tests from Step 1) + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/settings.rs +git commit -m "Add configurable options struct for the SSAT debug comment" +``` + +--- + +### Task 2: Thread options through publisher.rs redaction functions + +**Files:** +- Modify: `crates/trusted-server-core/src/publisher.rs:870-936` (allowlist const removal, `redact_response_for_dump`, `redact_bid_for_dump`) +- Modify: `crates/trusted-server-core/src/publisher.rs:950-1036` (`prepend_auction_debug_comment`) +- Modify: `crates/trusted-server-core/src/publisher.rs:1394-1396` (production call site) +- Modify: `crates/trusted-server-core/src/publisher.rs:2638` and `:2699` (existing test call sites) + +- [ ] **Step 1: Write failing tests for the new behavior** + +Add to the `publisher.rs` test module, near the existing `auction_debug_comment_*` tests (~line 2648 onward). These replace the fixed `dump_comment_for_creative` helper's implicit "always default options" behavior with an explicit parameter, so update the helper first: + +```rust +/// Build the ts-debug comment for a one-bid auction whose creative is +/// `creative`, so tests can assert on the rendered dump. +fn dump_comment_for_creative_with_options( + creative: &str, + options: &AuctionDebugCommentOptions, +) -> String { + let mut bid = make_test_bid_with_creative(creative); + bid.slot_id = "ad-header-0".to_string(); + let result = OrchestrationResult { + provider_responses: vec![ + AuctionResponse::no_bid("prebid", 665), + AuctionResponse::success("aps", vec![bid], 42), + ], + mediator_response: None, + winning_bids: std::collections::HashMap::new(), + total_time_ms: 665, + metadata: std::collections::HashMap::new(), + }; + let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); + prepend_auction_debug_comment("stream", &result, &state, options); + let comment = state + .lock() + .expect("should lock state") + .clone() + .expect("should have comment"); + drop(state); + comment +} + +fn dump_comment_for_creative(creative: &str) -> String { + dump_comment_for_creative_with_options(creative, &AuctionDebugCommentOptions::default()) +} + +#[test] +fn default_options_reproduce_current_behavior() { + // Identical to the pre-existing fixed output except: the unused `status` + // key (never written by any production path) is gone, and http_status / + // upstream_message / upstream_message_truncated are now allowlisted. + let comment = dump_comment_for_creative("
plain
"); + assert!(comment.contains("\"status\":\"nobid\"")); + assert!(comment.contains("dump={\"provider_responses\":")); + assert!(!comment.contains("mediator_response")); +} + +#[test] +fn metadata_keys_empty_yields_empty_metadata_object() { + let options = AuctionDebugCommentOptions { + metadata_keys: vec![], + ..AuctionDebugCommentOptions::default() + }; + let comment = dump_comment_for_creative_with_options("
x
", &options); + assert!( + comment.contains("\"metadata\":{}"), + "empty metadata_keys should yield an empty metadata object: {comment}" + ); +} + +#[test] +fn metadata_keys_attack_vector_debug_key_never_surfaces_in_redacted_mode() { + // Configuring "debug" in metadata_keys must have zero effect in Redacted + // mode — the allowlist intersection is the actual security boundary, not + // the config value. This is the load-bearing test for this whole design. + let response = AuctionResponse::error("prebid", 12).with_metadata( + "debug", + serde_json::json!({"resolvedrequest": {"user": {"id": "EC-ID-abc123"}}}), + ); + let result = OrchestrationResult { + provider_responses: vec![response], + mediator_response: None, + winning_bids: std::collections::HashMap::new(), + total_time_ms: 12, + metadata: std::collections::HashMap::new(), + }; + let options = AuctionDebugCommentOptions { + metadata_keys: vec!["debug".to_string()], + ..AuctionDebugCommentOptions::default() + }; + let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); + prepend_auction_debug_comment("stream", &result, &state, &options); + let comment = state.lock().expect("should lock state").clone().expect("should have comment"); + assert!( + !comment.contains("EC-ID-abc123"), + "debug key must never surface in Redacted mode even if configured: {comment}" + ); +} + +#[test] +fn verbosity_full_includes_raw_debug_subtree_when_present() { + let response = AuctionResponse::error("prebid", 12).with_metadata( + "debug", + serde_json::json!({"httpcalls": {"aps": [{"status": 200}]}}), + ); + let result = OrchestrationResult { + provider_responses: vec![response], + mediator_response: None, + winning_bids: std::collections::HashMap::new(), + total_time_ms: 12, + metadata: std::collections::HashMap::new(), + }; + let options = AuctionDebugCommentOptions { + verbosity: AuctionDebugCommentVerbosity::Full, + ..AuctionDebugCommentOptions::default() + }; + let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); + prepend_auction_debug_comment("stream", &result, &state, &options); + let comment = state.lock().expect("should lock state").clone().expect("should have comment"); + assert!( + comment.contains("httpcalls"), + "Full verbosity should surface the raw debug subtree: {comment}" + ); +} + +#[test] +fn verbosity_full_skips_creative_truncation() { + let big_creative = "y".repeat(MAX_BID_CREATIVE_DUMP_BYTES * 2); + let options = AuctionDebugCommentOptions { + verbosity: AuctionDebugCommentVerbosity::Full, + ..AuctionDebugCommentOptions::default() + }; + let comment = dump_comment_for_creative_with_options(&big_creative, &options); + assert!( + comment.contains(&big_creative), + "Full verbosity should not truncate the creative preview" + ); +} + +#[test] +fn verbosity_full_still_hits_overall_byte_cap() { + let huge_creative = "z".repeat(MAX_AUCTION_DEBUG_DUMP_BYTES * 2); + let options = AuctionDebugCommentOptions { + verbosity: AuctionDebugCommentVerbosity::Full, + ..AuctionDebugCommentOptions::default() + }; + let comment = dump_comment_for_creative_with_options(&huge_creative, &options); + assert!( + comment.contains("(truncated"), + "even Full verbosity must respect the total dump byte cap: {}", + &comment[..comment.len().min(200)] + ); +} + +#[test] +fn include_provider_responses_false_omits_section_entirely() { + let options = AuctionDebugCommentOptions { + include_provider_responses: false, + ..AuctionDebugCommentOptions::default() + }; + let comment = dump_comment_for_creative_with_options("
x
", &options); + assert!(!comment.contains("provider_responses")); +} + +#[test] +fn include_mediator_response_false_omits_even_when_mediator_ran() { + let response = AuctionResponse::success("aps", vec![], 10); + let mediator = AuctionResponse::success("mediator", vec![], 5); + let result = OrchestrationResult { + provider_responses: vec![response], + mediator_response: Some(mediator), + winning_bids: std::collections::HashMap::new(), + total_time_ms: 10, + metadata: std::collections::HashMap::new(), + }; + let options = AuctionDebugCommentOptions { + include_mediator_response: false, + ..AuctionDebugCommentOptions::default() + }; + let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); + prepend_auction_debug_comment("stream", &result, &state, &options); + let comment = state.lock().expect("should lock state").clone().expect("should have comment"); + assert!(!comment.contains("mediator_response")); +} + +#[test] +fn include_bids_false_yields_empty_bids_array_not_omitted_response() { + let options = AuctionDebugCommentOptions { + include_bids: false, + ..AuctionDebugCommentOptions::default() + }; + let comment = dump_comment_for_creative_with_options("
x
", &options); + assert!(comment.contains("\"bids\":[]")); + // The provider entry itself (status/provider name) must still be present. + assert!(comment.contains("\"provider\":\"aps\"")); +} + +#[test] +fn verbosity_full_still_neutralises_comment_terminators() { + let options = AuctionDebugCommentOptions { + verbosity: AuctionDebugCommentVerbosity::Full, + ..AuctionDebugCommentOptions::default() + }; + for creative in ["
evil-->break
", "--!>"] { + let comment = dump_comment_for_creative_with_options(creative, &options); + assert_eq!(comment.matches("-->").count(), 1); + assert!(!comment.contains("--!>")); + } +} +``` + +Also update the two pre-existing tests that call `prepend_auction_debug_comment` directly with the old 3-arg signature — `auction_debug_comment_dumps_provider_status` (uses the helper, already covered by the helper update above) and `auction_debug_comment_never_leaks_provider_debug_metadata` (~line 2699, calls the function directly): + +```rust + let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); + prepend_auction_debug_comment("stream", &result, &state, &AuctionDebugCommentOptions::default()); +``` + +Run: `cargo test-axum -p trusted-server-core --lib publisher:: -- --nocapture` +Expected: FAIL to compile — `prepend_auction_debug_comment` doesn't take a 4th argument yet; `redact_response_for_dump`/`redact_bid_for_dump` don't take `options` yet. + +- [ ] **Step 2: Remove the old local allowlist const, import the new one** + +Delete from `publisher.rs` (lines 870-886, the old `DEBUG_DUMP_METADATA_ALLOWLIST` const and its doc comment) and add near the top of the file's imports: + +```rust +use crate::settings::AUCTION_DEBUG_METADATA_ALLOWLIST; +use crate::settings::{AuctionDebugCommentOptions, AuctionDebugCommentVerbosity}; +``` + +- [ ] **Step 3: Update redact_bid_for_dump and redact_response_for_dump** + +Replace the two functions (publisher.rs ~905-936): + +```rust +/// Build a redacted JSON view of a single provider response for the +/// `ts-debug` dump. In [`AuctionDebugCommentVerbosity::Redacted`], only keys +/// in `options.metadata_keys ∩ AUCTION_DEBUG_METADATA_ALLOWLIST` survive and +/// each bid's creative is previewed to [`MAX_BID_CREATIVE_DUMP_BYTES`]. In +/// [`AuctionDebugCommentVerbosity::Full`], metadata and creatives pass +/// through unfiltered. +fn redact_response_for_dump( + response: &crate::auction::types::AuctionResponse, + options: &AuctionDebugCommentOptions, +) -> serde_json::Value { + let metadata: serde_json::Map = match options.verbosity { + AuctionDebugCommentVerbosity::Redacted => response + .metadata + .iter() + .filter(|(key, _)| { + options.metadata_keys.iter().any(|configured| configured == *key) + && AUCTION_DEBUG_METADATA_ALLOWLIST.contains(&key.as_str()) + }) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + AuctionDebugCommentVerbosity::Full => response + .metadata + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + }; + let bids: Vec = if options.include_bids { + response.bids.iter().map(|bid| redact_bid_for_dump(bid, options)).collect() + } else { + Vec::new() + }; + serde_json::json!({ + "provider": response.provider, + "status": response.status, + "response_time_ms": response.response_time_ms, + "bids": bids, + "metadata": metadata, + }) +} + +/// Build a redacted JSON view of a single bid. In `Redacted` verbosity, +/// `creative` is previewed to [`MAX_BID_CREATIVE_DUMP_BYTES`]; in `Full`, it +/// passes through untruncated. +fn redact_bid_for_dump( + bid: &crate::auction::types::Bid, + options: &AuctionDebugCommentOptions, +) -> serde_json::Value { + let mut value = serde_json::to_value(bid).unwrap_or(serde_json::Value::Null); + if options.verbosity == AuctionDebugCommentVerbosity::Redacted + && let Some(creative) = &bid.creative + { + value["creative"] = + serde_json::Value::String(truncate_with_marker(creative, MAX_BID_CREATIVE_DUMP_BYTES)); + } + value +} +``` + +Note the `metadata_keys.iter().any(...)` check: this is the intersection — a key must be BOTH configured AND in the hardcoded superset. `AUCTION_DEBUG_METADATA_ALLOWLIST.contains` alone would let an operator narrow but a bug in this line (e.g. only checking `metadata_keys`) would break the fail-closed guarantee. This is exactly what `metadata_keys_attack_vector_debug_key_never_surfaces_in_redacted_mode` (Task 2, Step 1) verifies. + +- [ ] **Step 4: Update prepend_auction_debug_comment** + +Replace the function body (publisher.rs ~950-1028) to add the `options` parameter and gate the two top-level sections: + +```rust +pub(crate) fn prepend_auction_debug_comment( + path_label: &str, + result: &crate::auction::orchestrator::OrchestrationResult, + ad_bids_state: &Arc>>, + options: &AuctionDebugCommentOptions, +) { + let ssp_count = result.provider_responses.len(); + let mediator_info = match &result.mediator_response { + Some(r) => format!("ok({}_bids)", r.bids.len()), + None => "none".to_string(), + }; + let mut dump = serde_json::Map::new(); + if options.include_provider_responses { + dump.insert( + "provider_responses".to_string(), + serde_json::Value::Array( + result + .provider_responses + .iter() + .map(|r| redact_response_for_dump(r, options)) + .collect(), + ), + ); + } + if options.include_mediator_response + && let Some(mediator_response) = &result.mediator_response + { + dump.insert( + "mediator_response".to_string(), + redact_response_for_dump(mediator_response, options), + ); + } + // ... rest of the function (render_dump closure, debug_comment format!, + // state locking) is UNCHANGED — do not modify below this point. +``` + +Everything from the `render_dump` closure onward (the neutralization + byte-cap logic, the `format!("` +HTML comment before the bids ` + +``` + +Do not alter: + +- publisher-originated DataDome script tags; +- `rewrite_sdk` behavior; +- the DataDome SDK proxy route; +- the signal collection API proxy; +- DataDome configuration serialization for non-suppressed requests; or +- injection behavior for requests without the marker. + +## Testing plan + +### Protection-filter tests + +Add or extend tests in +`crates/trusted-server-core/src/integrations/datadome/protection.rs` to verify +that the marker is attached for: + +- a matching inline IPv4 CIDR; +- a matching Config Store-backed CIDR source; +- a matching structured `ip_cidr` rule; and +- a matching structured `ip_cidr_source` rule. + +Verify that the marker is absent for: + +- a non-matching IP; +- an ASN exclusion; +- a path exclusion; +- a query-parameter exclusion; +- an excluded method; and +- an internal or integration route. + +Verify the existing protection behavior remains unchanged: IP-matched requests +continue without a Protection API call. + +### Head-injector tests + +Add tests in +`crates/trusted-server-core/src/integrations/datadome.rs` verifying that: + +- a configured client tag is omitted when suppression is active; +- a configured client tag is emitted when suppression is inactive; +- a blank client-side key remains a no-op; and +- `inject_client_side_tag = false` remains a no-op. + +### HTML pipeline tests + +Add coverage for the request-scoped value flowing through the HTML processor, +including the streaming path. Confirm that a suppressed processed HTML response +contains neither the injected `window.ddjskey` configuration nor the configured +DataDome `tags.js` script. For a suppressed HTML stream, assert the response is +private and has no surrogate cache headers. Confirm a non-suppressed HTML stream +retains its origin cache behavior. + +Confirm that publisher-originated DataDome tags remain in the output and are +still rewritten according to the existing `rewrite_sdk` behavior. + +### Fastly dispatch tests + +Add a Fastly adapter dispatch test with: + +- DataDome protection enabled; +- a client IP matching an inline exclusion; +- a configured client-side key; and +- an HTML publisher response. + +The test should verify that the request continues without a Protection API +call, the response includes the `client_tag=omitted` decision log through the +existing test logging seam where available, and the generated tag is absent. + +Also cover a non-excluded request to confirm the generated tag remains present. + +## Documentation changes + +Update `docs/guide/integrations/datadome.md` to state that IP-excluded Fastly +requests skip both: + +- server-side Protection API validation; and +- Trusted Server's automatic client-side tag injection. + +Document that this does not remove or disable publisher-originated DataDome +tags, and that non-IP exclusions do not automatically suppress the client-side +tag. + +No configuration template changes are required because this behavior has no +new setting. + +## Files expected to change + +- `crates/trusted-server-core/src/integrations/registry.rs` + - Support the internal request-scoped annotation mechanism. +- `crates/trusted-server-core/src/integrations/datadome.rs` + - Define the marker and conditionally suppress head injection. +- `crates/trusted-server-core/src/integrations/datadome/protection.rs` + - Attach the marker for IP-based scope skips and enrich the skip log. +- `crates/trusted-server-core/src/integrations/registry.rs` or the relevant + HTML context definition + - Carry the suppression decision into head injection. +- `crates/trusted-server-core/src/html_processor.rs` + - Carry the request-scoped value into HTML integration context. +- `crates/trusted-server-core/src/publisher.rs` + - Snapshot and propagate the request marker through response processing. +- `docs/guide/integrations/datadome.md` + - Document the behavior. +- Relevant unit and Fastly adapter test modules. + +The exact split between registry request annotations and HTML context plumbing +should remain minimal and should not introduce a new public configuration API. + +## Verification + +Implementation verification should use the repository's target-matched +commands: + +```bash +cargo fmt --all -- --check +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +``` + +No live production validation is required for this implementation task. Live +browser verification will be performed later through the deployment/testing +workflow. From fc963341a085da137212e5e2ee83ec07c462a082 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 5 Aug 2026 19:49:24 -0500 Subject: [PATCH 159/395] Add DataDome staging test bypass --- .../src/integrations/datadome.rs | 128 +++++++++++++- .../src/integrations/datadome/protection.rs | 161 +++++++++++++++++- docs/guide/integrations/datadome.md | 52 ++++-- 3 files changed, 328 insertions(+), 13 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index db5932fb2..4dd55b28f 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -61,7 +61,7 @@ use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::header; -use http::{Method, StatusCode}; +use http::{HeaderName, Method, StatusCode}; use regex::Regex; use serde::Deserialize; use serde_json::Value as JsonValue; @@ -77,6 +77,7 @@ use crate::integrations::{ collect_body_bounded, collect_response_bounded, ensure_integration_backend, }; use crate::platform::{PlatformHttpRequest, RuntimeServices}; +use crate::redacted::Redacted; use crate::settings::{IntegrationConfig, Settings}; mod protection; @@ -117,6 +118,27 @@ static DATADOME_URL_PATTERN: LazyLock = LazyLock::new(|| { .expect("DataDome URL rewrite regex should compile") }); +/// Temporary static-header bypass for server-side `DataDome` protection. +/// +/// This is intended only for an access-controlled staging environment. A +/// matching header bypasses the server-side Protection API and is removed +/// before the publisher origin receives the request. +#[derive(Debug, Default, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProtectionTestBypassConfig { + /// Enables the bypass. Defaults to disabled when the section is present. + #[serde(default)] + pub enabled: bool, + + /// Header name carrying the temporary bypass credential. + #[serde(default)] + pub header_name: String, + + /// Static credential expected in [`Self::header_name`]. + #[serde(default)] + pub credential: Redacted, +} + /// Configuration for `DataDome` integration. #[derive(Debug, Clone, Deserialize, Validate)] #[serde(deny_unknown_fields)] @@ -199,6 +221,10 @@ pub struct DataDomeConfig { )] pub protection_exclusion_rules: Vec, + /// Temporary static-header bypass for access-controlled staging tests. + #[serde(default)] + pub protection_test_bypass: Option, + /// Reserved flag for future GraphQL payload extraction. #[serde(default)] pub enable_graphql_support: bool, @@ -329,6 +355,7 @@ impl Default for DataDomeConfig { protection_excluded_ip_cidr_sources: Vec::new(), protection_ip_list_cache_ttl_seconds: default_protection_ip_list_cache_ttl_seconds(), protection_exclusion_rules: default_protection_exclusion_rules(), + protection_test_bypass: None, enable_graphql_support: false, client_side_key: String::new(), inject_client_side_tag: default_inject_client_side_tag(), @@ -362,6 +389,9 @@ impl DataDomeIntegration { config.server_side_key_secret_name = config.server_side_key_secret_name.trim().to_string(); config.protection_api_origin = config.protection_api_origin.trim().to_string(); config.client_side_tag_url = config.client_side_tag_url.trim().to_string(); + if let Some(bypass) = &mut config.protection_test_bypass { + bypass.header_name = bypass.header_name.trim().to_string(); + } if config.enable_protection { if config.server_side_key_secret_store.is_empty() @@ -373,6 +403,7 @@ impl DataDomeIntegration { } Self::validate_protection_api_origin(&config.protection_api_origin)?; } + Self::validate_protection_test_bypass(&config)?; if config.inject_client_side_tag { Self::validate_client_side_tag_url(&config.client_side_tag_url)?; @@ -422,6 +453,36 @@ impl DataDomeIntegration { Ok(()) } + fn validate_protection_test_bypass( + config: &DataDomeConfig, + ) -> Result<(), Report> { + let Some(bypass) = config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + else { + return Ok(()); + }; + + if !config.enable_protection { + return Err(Report::new(Self::error( + "protection_test_bypass requires enable_protection to be true", + ))); + } + if HeaderName::from_bytes(bypass.header_name.as_bytes()).is_err() { + return Err(Report::new(Self::error( + "protection_test_bypass.header_name must be a valid HTTP header name", + ))); + } + if bypass.credential.expose().is_empty() { + return Err(Report::new(Self::error( + "protection_test_bypass.credential must not be empty when enabled", + ))); + } + + Ok(()) + } + fn validate_client_side_tag_url(tag_url: &str) -> Result<(), Report> { if tag_url.starts_with('/') && !tag_url.starts_with("//") { if tag_url.chars().any(is_unsafe_client_side_tag_path_char) { @@ -1098,6 +1159,71 @@ mod tests { config.server_side_key_secret_name, "datadome_server_side_key" ); + assert!( + config.protection_test_bypass.is_none(), + "the temporary test bypass should be disabled by default" + ); + } + + #[test] + fn protection_test_bypass_deserializes_nested_configuration() { + let config: DataDomeConfig = toml::from_str( + r#" + enabled = true + enable_protection = true + + [protection_test_bypass] + enabled = true + header_name = "x-ts-datadome-test-bypass" + credential = "temporary-test-credential" + "#, + ) + .expect("should deserialize DataDome test bypass configuration"); + let bypass = config + .protection_test_bypass + .expect("should deserialize the nested test bypass configuration"); + + assert!(bypass.enabled, "should retain the enabled flag"); + assert_eq!( + bypass.header_name, "x-ts-datadome-test-bypass", + "should retain the configured header name" + ); + assert_eq!( + bypass.credential.expose(), + "temporary-test-credential", + "should retain the configured credential" + ); + } + + #[test] + fn protection_test_bypass_requires_protection_header_and_credential() { + for (enable_protection, header_name, credential, expected_message) in [ + ( + false, + "x-ts-datadome-test-bypass", + "temporary-test-credential", + "requires enable_protection", + ), + (true, "", "temporary-test-credential", "header_name"), + (true, "x-ts-datadome-test-bypass", "", "credential"), + ] { + let mut config = test_config(); + config.enable_protection = enable_protection; + config.protection_test_bypass = Some(ProtectionTestBypassConfig { + enabled: true, + header_name: header_name.to_string(), + credential: Redacted::new(credential.to_string()), + }); + + let err = match DataDomeIntegration::try_new(config) { + Ok(_) => panic!("should reject invalid protection test bypass configuration"), + Err(err) => err, + }; + assert!( + format!("{err:?}").contains(expected_message), + "should explain the invalid protection test bypass configuration" + ); + } } #[test] diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 4c74c71e8..e67a83e36 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -4,6 +4,8 @@ use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::{HeaderMap, HeaderName, request_builder}; use error_stack::{Report, ResultExt}; use http::{Method, Request, Response, StatusCode, header}; +use sha2::{Digest as _, Sha256}; +use subtle::ConstantTimeEq as _; use url::Url; use crate::error::TrustedServerError; @@ -45,10 +47,20 @@ impl DataDomeIntegration { ); } + let test_bypass_matched = self.take_protection_test_bypass_header(input.request); if !self.config.enable_protection || !self.is_request_protected(&mut input) { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } + if test_bypass_matched { + input + .request + .extensions_mut() + .insert(super::DataDomeClientTagSuppressed); + log_protection_test_bypass(&input); + return RequestFilterDecision::Continue(RequestFilterEffects::default()); + } + match self.filter_protection_request_inner(input).await { Ok(decision) => decision, Err(ProtectionRequestError::Setup(err)) => { @@ -156,6 +168,26 @@ impl DataDomeIntegration { true } + fn take_protection_test_bypass_header(&self, req: &mut Request) -> bool { + let Some(bypass) = self + .config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + else { + return false; + }; + let header_name = HeaderName::from_bytes(bypass.header_name.as_bytes()) + .expect("should validate protection test bypass header name during setup"); + let Some(value) = req.headers_mut().remove(&header_name) else { + return false; + }; + + let actual = Sha256::digest(value.as_bytes()); + let expected = Sha256::digest(bypass.credential.expose().as_bytes()); + bool::from(actual.ct_eq(&expected)) + } + fn protection_validate_url(&self) -> String { format!( "{}{}", @@ -442,6 +474,15 @@ fn is_ip_exclusion_reason(reason: &str) -> bool { ) } +fn log_protection_test_bypass(input: &RequestFilterInput<'_>) { + log::info!( + "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={} host={} path={}", + input.request.method(), + request_host(input.request), + input.request.uri().path(), + ); +} + fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { if is_ip_exclusion_reason(reason) { log::info!( @@ -734,12 +775,13 @@ mod tests { use crate::integrations::datadome::{ DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, + ProtectionTestBypassConfig, }; use crate::platform::GeoInfo; use crate::platform::test_support::{ - HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopSecretStore, + HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopSecretStore, StubHttpClient, build_services_with_config_and_secret, build_services_with_config_and_secret_and_client_ip, - noop_services_with_client_ip, + build_services_with_secret_and_http_client, noop_services_with_client_ip, }; use crate::settings::Settings; @@ -801,6 +843,121 @@ mod tests { .is_some() } + #[test] + fn protection_test_bypass_skips_api_suppresses_tag_and_strips_header() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + header_name: "x-ts-datadome-test-bypass".to_string(), + credential: Redacted::new("temporary-test-credential".to_string()), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let http_client = Arc::new(StubHttpClient::new()); + let services = + build_services_with_secret_and_http_client(NoopSecretStore, http_client.clone()); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + "x-ts-datadome-test-bypass", + edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "a matching test credential should continue without a challenge" + ); + assert!( + has_client_tag_suppression_marker(&request), + "the bypass should suppress the automatic DataDome client tag" + ); + assert!( + request.headers().get("x-ts-datadome-test-bypass").is_none(), + "the bypass credential must not reach the publisher origin" + ); + assert!( + http_client.recorded_backend_names().is_empty(), + "a matching test credential must not call the Protection API" + ); + } + + #[test] + fn protection_test_bypass_strips_invalid_credential_without_bypassing() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + header_name: "x-ts-datadome-test-bypass".to_string(), + credential: Redacted::new("temporary-test-credential".to_string()), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + "x-ts-datadome-test-bypass", + edgezero_core::http::HeaderValue::from_static("wrong-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an allowed Protection API response should continue" + ); + assert!( + !has_client_tag_suppression_marker(&request), + "a non-matching credential must not suppress the DataDome client tag" + ); + assert!( + request.headers().get("x-ts-datadome-test-bypass").is_none(), + "an invalid bypass credential must not reach the publisher origin" + ); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "a non-matching credential must still call the Protection API" + ); + } + #[test] fn ip_exclusions_mark_requests_for_client_tag_suppression() { let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 1743a1b75..ba3081f2a 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -86,6 +86,7 @@ patterns = ["(?i)\\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav| | `protection_excluded_ip_cidr_sources` | array | `[]` | Config Store sources containing dynamic client IP CIDR bypass lists | | `protection_ip_list_cache_ttl_seconds` | integer | `300` | Process-local cache TTL for Config Store-backed IP CIDR bypass lists | | `protection_exclusion_rules` | array | Static asset path regex | Structured method/path/query/IP/ASN exclusion rules | +| `protection_test_bypass` | object | omitted | Temporary static-header bypass for access-controlled staging tests | | `enable_graphql_support` | boolean | `false` | Reserved for future GraphQL body inspection; ignored in v1 | | `client_side_key` | string | `""` | DataDome client-side JavaScript key used for tag injection | | `inject_client_side_tag` | boolean | `true` | Auto-inject the browser tag when `client_side_key` is non-empty | @@ -168,36 +169,67 @@ A request is protected when all of the following are true: 5. The client IP does not match `protection_excluded_ip_cidrs` or any Config Store-backed CIDR source. 6. The client ASN is not listed in `protection_excluded_asns`. 7. No `protection_exclusion_rules` match. +8. The request does not contain a matching enabled `protection_test_bypass` credential. Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, admin routes, discovery routes, and signature-verification routes are also excluded by default. Auction traffic at `/auction` is protected by default. -### IP-excluded client-side tag behavior +### Staging test bypass + +For short-lived browser automation on an access-controlled staging site, you +can configure a static header credential that skips only the server-side +Protection API: + +```toml +[integrations.datadome.protection_test_bypass] +enabled = true +header_name = "x-ts-datadome-test-bypass" +credential = "temporary-test-credential" +``` + +`protection_test_bypass` requires `enable_protection = true`; it is disabled +when omitted. Treat the credential as a temporary secret: configure it only +while needed, protect the site with an outer access control such as Basic Auth, +and remove the section when testing finishes. Do not enable it in production. + +A matching header is compared in constant time, removed before the request can +reach DataDome or the publisher origin, and never logged. With Playwright, +apply it to the browser context: + +```ts +await context.setExtraHTTPHeaders({ + "X-TS-DataDome-Test-Bypass": process.env.DATADOME_TEST_BYPASS!, +}); +``` + +### Client-side tag suppression behavior On the Fastly adapter, a request that matches an IP-based DataDome exclusion -also omits Trusted Server's automatically injected client-side DataDome tag -from processed HTML. This keeps the client-side layer consistent with the -server-side Protection API skip. +or the configured test-bypass credential also omits Trusted Server's +automatically injected client-side DataDome tag from processed HTML. This keeps +the client-side layer consistent with the server-side Protection API skip. This behavior applies to: - `protection_excluded_ip_cidrs`; - `protection_excluded_ip_cidr_sources`; -- structured `ip_cidr` rules; and -- structured `ip_cidr_source` rules. +- structured `ip_cidr` rules; +- structured `ip_cidr_source` rules; and +- a matching enabled `protection_test_bypass` credential. ASN, method, path, query-parameter, static-asset, and internal-route exclusions do not automatically suppress the client-side tag. DataDome tags already present in publisher HTML are not removed or changed by this behavior, and `/integrations/datadome/tags.js` remains available when requested directly. -Because the processed HTML differs by client IP, tag-suppressed HTML is marked -`private, max-age=0` and removed from shared surrogate caches. The decision is -reported in the existing protection log, for example: +Because the processed HTML differs by client IP or test credential, +tag-suppressed HTML is marked `private, max-age=0` and removed from shared +surrogate caches. The decision is reported in the existing protection log, for +example: ```text -[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET host=example.com path=/page ``` ### Structured exclusion rules From 1ce7892b145d3643a0e916311291035759d3e7ea Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 10:49:17 -0500 Subject: [PATCH 160/395] Log DataDome test-bypass registration state Log whether protection_test_bypass is enabled when registering the DataDome integration and include configured header name when enabled. Keep credential secret out of logs., --- .../src/integrations/datadome.rs | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 4dd55b28f..112d41da1 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -914,12 +914,26 @@ fn build( return Ok(None); }; - log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {})", - config.sdk_origin, - config.rewrite_sdk, - config.enable_protection - ); + if let Some(bypass) = config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + { + log::info!( + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: enabled, protection_test_bypass_header: {})", + config.sdk_origin, + config.rewrite_sdk, + config.enable_protection, + bypass.header_name, + ); + } else { + log::info!( + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: disabled)", + config.sdk_origin, + config.rewrite_sdk, + config.enable_protection, + ); + } Ok(Some(DataDomeIntegration::try_new(config)?)) } From 7e6f365ad04b329dfa138cac0736c12220ebb8be Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 12:49:49 -0500 Subject: [PATCH 161/395] fix datadome staging bypass privacy --- crates/trusted-server-core/src/config.rs | 42 ++++- .../src/integrations/datadome.rs | 111 +++++------ .../src/integrations/datadome/protection.rs | 173 +++++++++++++----- crates/trusted-server-core/src/publisher.rs | 102 ++++++++++- .../src/response_privacy.rs | 3 +- docs/guide/integrations/datadome.md | 31 ++-- ...6-08-03-datadome-ip-excluded-client-tag.md | 2 +- ...-datadome-ip-excluded-client-tag-design.md | 4 +- 8 files changed, 347 insertions(+), 121 deletions(-) diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index cdee3b222..e74ef4150 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -155,7 +155,9 @@ fn validate_enabled_integrations( validate_integration::(settings, "sourcepoint")?; validate_integration::(settings, "osano")?; validate_integration::(settings, "google_tag_manager")?; - validate_integration::(settings, "datadome")?; + if let Some(config) = settings.integration_config::("datadome")? { + crate::integrations::datadome::DataDomeIntegration::validate_config_for_startup(config)?; + } validate_integration::(settings, "gpt")?; validate_integration::(settings, "gpt_diagnostics")?; @@ -404,6 +406,44 @@ password = "production-admin-password-32-bytes" ); } + #[test] + fn deploy_validation_rejects_invalid_datadome_test_bypass() { + for (enable_protection, store, name, expected_message) in [ + ( + false, + "ts_secrets", + "datadome_test_bypass", + "requires enable_protection", + ), + (true, "", "datadome_test_bypass", "credential_secret_store"), + (true, "ts_secrets", "", "credential_secret_name"), + ] { + let mut settings = valid_settings(); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "enable_protection": enable_protection, + "protection_test_bypass": { + "enabled": true, + "credential_secret_store": store, + "credential_secret_name": name, + }, + }), + ) + .expect("should insert DataDome config"); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject invalid DataDome test bypass"); + assert!( + format!("{err:?}").contains(expected_message), + "error should mention the invalid bypass setting: {err:?}" + ); + } + } + #[test] fn validate_trait_reports_deploy_errors() { let mut settings = valid_settings(); diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 112d41da1..2486c16be 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -61,7 +61,7 @@ use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::header; -use http::{HeaderName, Method, StatusCode}; +use http::{Method, StatusCode}; use regex::Regex; use serde::Deserialize; use serde_json::Value as JsonValue; @@ -77,7 +77,6 @@ use crate::integrations::{ collect_body_bounded, collect_response_bounded, ensure_integration_backend, }; use crate::platform::{PlatformHttpRequest, RuntimeServices}; -use crate::redacted::Redacted; use crate::settings::{IntegrationConfig, Settings}; mod protection; @@ -90,6 +89,7 @@ pub use protection_scope::{ use protection_scope::ProtectionScope; pub(crate) const DATADOME_INTEGRATION_ID: &str = "datadome"; +pub(crate) const HEADER_DATADOME_TEST_BYPASS: &str = "x-ts-datadome-bypass"; /// Request marker indicating that Trusted Server should omit its automatic /// `DataDome` client-side tag for the current response. @@ -121,8 +121,9 @@ static DATADOME_URL_PATTERN: LazyLock = LazyLock::new(|| { /// Temporary static-header bypass for server-side `DataDome` protection. /// /// This is intended only for an access-controlled staging environment. A -/// matching header bypasses the server-side Protection API and is removed -/// before the publisher origin receives the request. +/// matching `x-ts-datadome-bypass` header bypasses the server-side Protection +/// API and is removed before the publisher origin receives the request. The +/// credential itself is loaded from the Secret Store at runtime. #[derive(Debug, Default, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct ProtectionTestBypassConfig { @@ -130,13 +131,13 @@ pub struct ProtectionTestBypassConfig { #[serde(default)] pub enabled: bool, - /// Header name carrying the temporary bypass credential. - #[serde(default)] - pub header_name: String, + /// Secret Store containing the temporary bypass credential. + #[serde(default = "default_protection_test_bypass_secret_store")] + pub credential_secret_store: String, - /// Static credential expected in [`Self::header_name`]. - #[serde(default)] - pub credential: Redacted, + /// Secret name containing the temporary bypass credential. + #[serde(default = "default_protection_test_bypass_secret_name")] + pub credential_secret_name: String, } /// Configuration for `DataDome` integration. @@ -278,6 +279,14 @@ fn default_server_side_key_secret_name() -> String { "datadome_server_side_key".to_string() } +fn default_protection_test_bypass_secret_store() -> String { + "ts_secrets".to_string() +} + +fn default_protection_test_bypass_secret_name() -> String { + "datadome_test_bypass".to_string() +} + fn default_timeout_ms() -> u32 { 1500 } @@ -390,7 +399,8 @@ impl DataDomeIntegration { config.protection_api_origin = config.protection_api_origin.trim().to_string(); config.client_side_tag_url = config.client_side_tag_url.trim().to_string(); if let Some(bypass) = &mut config.protection_test_bypass { - bypass.header_name = bypass.header_name.trim().to_string(); + bypass.credential_secret_store = bypass.credential_secret_store.trim().to_string(); + bypass.credential_secret_name = bypass.credential_secret_name.trim().to_string(); } if config.enable_protection { @@ -453,6 +463,12 @@ impl DataDomeIntegration { Ok(()) } + pub(crate) fn validate_config_for_startup( + config: DataDomeConfig, + ) -> Result<(), Report> { + Self::try_new(config).map(|_| ()) + } + fn validate_protection_test_bypass( config: &DataDomeConfig, ) -> Result<(), Report> { @@ -469,14 +485,9 @@ impl DataDomeIntegration { "protection_test_bypass requires enable_protection to be true", ))); } - if HeaderName::from_bytes(bypass.header_name.as_bytes()).is_err() { + if bypass.credential_secret_store.is_empty() || bypass.credential_secret_name.is_empty() { return Err(Report::new(Self::error( - "protection_test_bypass.header_name must be a valid HTTP header name", - ))); - } - if bypass.credential.expose().is_empty() { - return Err(Report::new(Self::error( - "protection_test_bypass.credential must not be empty when enabled", + "protection_test_bypass credential_secret_store and credential_secret_name must not be empty when enabled", ))); } @@ -914,28 +925,25 @@ fn build( return Ok(None); }; - if let Some(bypass) = config + let integration = DataDomeIntegration::try_new(config)?; + let protection_test_bypass = integration + .config .protection_test_bypass .as_ref() - .filter(|bypass| bypass.enabled) - { - log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: enabled, protection_test_bypass_header: {})", - config.sdk_origin, - config.rewrite_sdk, - config.enable_protection, - bypass.header_name, - ); - } else { - log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: disabled)", - config.sdk_origin, - config.rewrite_sdk, - config.enable_protection, - ); - } + .is_some_and(|bypass| bypass.enabled); + log::info!( + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: {})", + integration.config.sdk_origin, + integration.config.rewrite_sdk, + integration.config.enable_protection, + if protection_test_bypass { + "enabled" + } else { + "disabled" + }, + ); - Ok(Some(DataDomeIntegration::try_new(config)?)) + Ok(Some(integration)) } /// Register the `DataDome` integration with Trusted Server. @@ -1188,8 +1196,8 @@ mod tests { [protection_test_bypass] enabled = true - header_name = "x-ts-datadome-test-bypass" - credential = "temporary-test-credential" + credential_secret_store = "ts_secrets" + credential_secret_name = "datadome_test_bypass" "#, ) .expect("should deserialize DataDome test bypass configuration"); @@ -1199,34 +1207,33 @@ mod tests { assert!(bypass.enabled, "should retain the enabled flag"); assert_eq!( - bypass.header_name, "x-ts-datadome-test-bypass", - "should retain the configured header name" + bypass.credential_secret_store, "ts_secrets", + "should retain the configured credential Secret Store" ); assert_eq!( - bypass.credential.expose(), - "temporary-test-credential", - "should retain the configured credential" + bypass.credential_secret_name, "datadome_test_bypass", + "should retain the configured credential secret name" ); } #[test] - fn protection_test_bypass_requires_protection_header_and_credential() { - for (enable_protection, header_name, credential, expected_message) in [ + fn protection_test_bypass_requires_protection_and_secret_references() { + for (enable_protection, store, name, expected_message) in [ ( false, - "x-ts-datadome-test-bypass", - "temporary-test-credential", + "ts_secrets", + "datadome_test_bypass", "requires enable_protection", ), - (true, "", "temporary-test-credential", "header_name"), - (true, "x-ts-datadome-test-bypass", "", "credential"), + (true, "", "datadome_test_bypass", "credential_secret_store"), + (true, "ts_secrets", "", "credential_secret_name"), ] { let mut config = test_config(); config.enable_protection = enable_protection; config.protection_test_bypass = Some(ProtectionTestBypassConfig { enabled: true, - header_name: header_name.to_string(), - credential: Redacted::new(credential.to_string()), + credential_secret_store: store.to_string(), + credential_secret_name: name.to_string(), }); let err = match DataDomeIntegration::try_new(config) { diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index e67a83e36..bc06214ff 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -38,20 +38,8 @@ impl DataDomeIntegration { &self, mut input: RequestFilterInput<'_>, ) -> RequestFilterDecision { - if self.config.enable_protection { - log::info!( - "[datadome] protection incoming method={} host={} path={}", - input.request.method(), - request_host(input.request), - input.request.uri().path(), - ); - } - - let test_bypass_matched = self.take_protection_test_bypass_header(input.request); - if !self.config.enable_protection || !self.is_request_protected(&mut input) { - return RequestFilterDecision::Continue(RequestFilterEffects::default()); - } - + let test_bypass_matched = + self.take_protection_test_bypass_header(input.request, input.services); if test_bypass_matched { input .request @@ -61,6 +49,10 @@ impl DataDomeIntegration { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } + if !self.config.enable_protection || !self.is_request_protected(&mut input) { + return RequestFilterDecision::Continue(RequestFilterEffects::default()); + } + match self.filter_protection_request_inner(input).await { Ok(decision) => decision, Err(ProtectionRequestError::Setup(err)) => { @@ -168,7 +160,11 @@ impl DataDomeIntegration { true } - fn take_protection_test_bypass_header(&self, req: &mut Request) -> bool { + fn take_protection_test_bypass_header( + &self, + req: &mut Request, + services: &RuntimeServices, + ) -> bool { let Some(bypass) = self .config .protection_test_bypass @@ -177,14 +173,32 @@ impl DataDomeIntegration { else { return false; }; - let header_name = HeaderName::from_bytes(bypass.header_name.as_bytes()) - .expect("should validate protection test bypass header name during setup"); - let Some(value) = req.headers_mut().remove(&header_name) else { + let Some(value) = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS) else { return false; }; + let store_name = StoreName::from(bypass.credential_secret_store.as_str()); + let credential = match services + .secret_store() + .get_string(&store_name, &bypass.credential_secret_name) + { + Ok(credential) if !credential.is_empty() => credential, + Ok(_) => { + log::warn!( + "[datadome] DataDome test bypass credential is empty; ignoring bypass header" + ); + return false; + } + Err(err) => { + log::warn!( + "[datadome] Failed to load DataDome test bypass credential; ignoring bypass header: {err:?}" + ); + return false; + } + }; + let actual = Sha256::digest(value.as_bytes()); - let expected = Sha256::digest(bypass.credential.expose().as_bytes()); + let expected = Sha256::digest(credential.as_bytes()); bool::from(actual.ct_eq(&expected)) } @@ -476,31 +490,25 @@ fn is_ip_exclusion_reason(reason: &str) -> bool { fn log_protection_test_bypass(input: &RequestFilterInput<'_>) { log::info!( - "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={} host={} path={}", + "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={}", input.request.method(), - request_host(input.request), - input.request.uri().path(), ); } fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { if is_ip_exclusion_reason(reason) { log::info!( - "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={} host={} path={}", + "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", rule_id, reason, input.request.method(), - request_host(input.request), - input.request.uri().path(), ); } else { log::debug!( - "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={}", + "[datadome] protection decision=skipped rule={} reason={} method={}", rule_id, reason, input.request.method(), - request_host(input.request), - input.request.uri().path(), ); } } @@ -512,26 +520,20 @@ fn log_protection_result( decision: &RequestFilterDecision, ) { let method = input.request.method(); - let host = request_host(input.request); - let path = input.request.uri().path(); match decision { RequestFilterDecision::Respond { .. } => log::info!( - "[datadome] protection decision=blocked status={} method={} host={} path={} route=short_circuit", + "[datadome] protection decision=blocked status={} method={} route=short_circuit", status.as_u16(), method, - host, - path, ), RequestFilterDecision::Continue(_) if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => { log::info!( - "[datadome] protection decision=allowed status={} method={} host={} path={} route=continue", + "[datadome] protection decision=allowed status={} method={} route=continue", status.as_u16(), method, - host, - path, ); } RequestFilterDecision::Continue(_) => {} @@ -850,19 +852,26 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - header_name: "x-ts-datadome-test-bypass".to_string(), - credential: Redacted::new("temporary-test-credential".to_string()), + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), }), ..DataDomeConfig::default() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); let http_client = Arc::new(StubHttpClient::new()); - let services = - build_services_with_secret_and_http_client(NoopSecretStore, http_client.clone()); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( - "x-ts-datadome-test-bypass", + super::super::HEADER_DATADOME_TEST_BYPASS, edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), ); @@ -885,7 +894,10 @@ mod tests { "the bypass should suppress the automatic DataDome client tag" ); assert!( - request.headers().get("x-ts-datadome-test-bypass").is_none(), + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), "the bypass credential must not reach the publisher origin" ); assert!( @@ -894,6 +906,68 @@ mod tests { ); } + #[test] + fn protection_test_bypass_wins_over_other_exclusions() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "staging-page-exclusion".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::PathExact { + paths: vec!["/page".to_string()], + }, + }], + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "a matching test credential should continue" + ); + assert!( + has_client_tag_suppression_marker(&request), + "a matching test credential should suppress the tag even on an excluded path" + ); + assert!( + http_client.recorded_backend_names().is_empty(), + "a matching test credential must not call the Protection API" + ); + } + #[test] fn protection_test_bypass_strips_invalid_credential_without_bypassing() { let config = DataDomeConfig { @@ -901,8 +975,8 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - header_name: "x-ts-datadome-test-bypass".to_string(), - credential: Redacted::new("temporary-test-credential".to_string()), + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), }), ..DataDomeConfig::default() }; @@ -912,6 +986,10 @@ mod tests { "datadome_server_side_key".to_string(), b"server-side-key".to_vec(), ); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); let http_client = Arc::new(StubHttpClient::new()); http_client.push_response_with_headers( 200, @@ -925,7 +1003,7 @@ mod tests { let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( - "x-ts-datadome-test-bypass", + super::super::HEADER_DATADOME_TEST_BYPASS, edgezero_core::http::HeaderValue::from_static("wrong-credential"), ); @@ -948,7 +1026,10 @@ mod tests { "a non-matching credential must not suppress the DataDome client tag" ); assert!( - request.headers().get("x-ts-datadome-test-bypass").is_none(), + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), "an invalid bypass credential must not reach the publisher origin" ); assert_eq!( diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 48e0eaacf..5fc8dac0e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1452,12 +1452,21 @@ fn apply_datadome_client_tag_cache_privacy( return; } - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), - ); - response.headers_mut().remove("surrogate-control"); - response.headers_mut().remove("fastly-surrogate-control"); + let already_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|value| value.contains("private") || value.contains("no-store")); + if !already_uncacheable { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=0"), + ); + } + for header_name in CDN_CACHE_HEADERS { + response.headers_mut().remove(*header_name); + } } /// Drop a bodiless response's body and correct its framing headers. @@ -2895,6 +2904,10 @@ pub async fn handle_publisher_request( .extensions() .get::() .is_some(); + if suppress_datadome_client_side_tag { + req.headers_mut().remove(header::IF_NONE_MATCH); + req.headers_mut().remove(header::IF_MODIFIED_SINCE); + } let mut platform_request = PlatformHttpRequest::new(req, backend_name); if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); @@ -5203,6 +5216,50 @@ mod tests { ); } + #[tokio::test] + async fn suppressed_publisher_request_removes_conditional_validators() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header(header::IF_NONE_MATCH, "\"cached-page\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .body(EdgeBody::empty()) + .expect("should build conditional request"); + req.extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_NONE_MATCH.as_str())), + "suppressed requests must not forward If-None-Match" + ); + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_MODIFIED_SINCE.as_str())), + "suppressed requests must not forward If-Modified-Since" + ); + } + #[tokio::test] async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { let settings = create_test_settings(); @@ -5365,6 +5422,8 @@ mod tests { .header(header::CACHE_CONTROL, "public, max-age=600") .header("surrogate-control", "max-age=600") .header("fastly-surrogate-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") .body(EdgeBody::empty()) .expect("should build cacheable HTML response"); @@ -5391,6 +5450,37 @@ mod tests { response.headers().get("fastly-surrogate-control").is_none(), "suppressed HTML should not retain Fastly-Surrogate-Control" ); + assert!( + response + .headers() + .get("cloudflare-cdn-cache-control") + .is_none(), + "suppressed HTML should not retain Cloudflare-CDN-Cache-Control" + ); + assert!( + response.headers().get("cdn-cache-control").is_none(), + "suppressed HTML should not retain CDN-Cache-Control" + ); + + let mut no_store_response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .body(EdgeBody::empty()) + .expect("should build no-store HTML response"); + super::apply_datadome_client_tag_cache_privacy( + &mut no_store_response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + assert_eq!( + no_store_response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store"), + "suppressed HTML should preserve an existing no-store policy" + ); } #[test] diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index e23348211..2205ae892 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -22,6 +22,7 @@ pub const CDN_CACHE_HEADERS: &[&str] = &[ "fastly-surrogate-control", "cdn-cache-control", "cloudflare-cdn-cache-control", + "cdn-cache-control", ]; /// Forces cookie-bearing responses to stay private to shared caches. @@ -37,7 +38,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { if !response.headers().contains_key(header::SET_COOKIE) { return; } - // Surrogate cache headers must come off every cookie-bearing response, even + // Shared-cache control headers must come off every cookie-bearing response, even // one already carrying a stricter `no-store`/`private` directive — they are // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index ba3081f2a..35eda5d3e 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -184,23 +184,30 @@ Protection API: ```toml [integrations.datadome.protection_test_bypass] enabled = true -header_name = "x-ts-datadome-test-bypass" -credential = "temporary-test-credential" +credential_secret_store = "ts_secrets" +credential_secret_name = "datadome_test_bypass" ``` `protection_test_bypass` requires `enable_protection = true`; it is disabled -when omitted. Treat the credential as a temporary secret: configure it only -while needed, protect the site with an outer access control such as Basic Auth, -and remove the section when testing finishes. Do not enable it in production. +when omitted. Store the temporary credential in the configured Secret Store, +configure this section only while needed, protect the site with an outer access +control such as Basic Auth, and remove the section when testing finishes. Do not +enable it in production. -A matching header is compared in constant time, removed before the request can -reach DataDome or the publisher origin, and never logged. With Playwright, -apply it to the browser context: +The fixed `x-ts-datadome-bypass` header is compared in constant time, removed +before the request can reach DataDome or the publisher origin, and never +logged. Scope the header to the staging origin; do not attach it to every +request in a browser context because that can disclose the credential to +third-party origins. With Playwright: ```ts -await context.setExtraHTTPHeaders({ - "X-TS-DataDome-Test-Bypass": process.env.DATADOME_TEST_BYPASS!, -}); +await context.route('https://staging.example.com/**', async (route) => { + const headers = { + ...route.request().headers(), + 'x-ts-datadome-bypass': process.env.DATADOME_TEST_BYPASS!, + } + await route.continue({ headers }) +}) ``` ### Client-side tag suppression behavior @@ -229,7 +236,7 @@ surrogate caches. The decision is reported in the existing protection log, for example: ```text -[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET ``` ### Structured exclusion rules diff --git a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md index 5ea5f6ff3..0dfc923cf 100644 --- a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md +++ b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md @@ -152,7 +152,7 @@ matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" desired shape is: ```text -[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET ``` - [ ] **Step 5: Add filter-level marker tests.** Add small helpers in the diff --git a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md index c48ad2fda..d6d811780 100644 --- a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md +++ b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md @@ -149,10 +149,10 @@ For IP-based skips, extend the existing informational log with `client_tag=omitted`: ```text -[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET ``` -The existing rule ID, reason, and request metadata remain part of the log. +The existing rule ID, reason, and method remain part of the log. Host and path are omitted to avoid placing dynamic request data in the protection logs. Client IP values are not included. Non-IP skip logs retain their current behavior and level. From 126c17fc73437adc7869d1e512901678eadec223 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 14:56:44 -0500 Subject: [PATCH 162/395] Improve publisher HTML cache policy when SSAT is inactive --- crates/trusted-server-core/src/publisher.rs | 105 ++++++++++++++------ 1 file changed, 77 insertions(+), 28 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d3410b4ed..8919f09b3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -2932,33 +2932,49 @@ pub async fn handle_publisher_request( // // Gate on `should_run_ad_stack` rather than content-type alone: when no slot // matched, the feature is disabled, or this is not an ad-eligible navigation, - // no per-user `tsjs.adSlots`/`tsjs.bids` are injected, so forcing private - // here would needlessly strip shared cacheability from ordinary publisher - // HTML. Applies regardless of the auction *outcome* (empty bids still inject - // per-user slot state). The separate EC-cookie cache net in the adapter's - // `finalize_response` keeps first-visit identity responses private. + // no per-user `tsjs.adSlots`/`tsjs.bids` are injected. Applies regardless of + // the auction *outcome* (empty bids still inject per-user slot state). The + // separate EC-cookie cache net in the adapter's `finalize_response` keeps + // first-visit identity responses private. let origin_content_type = response .headers() .get(header::CONTENT_TYPE) .and_then(|h| h.to_str().ok()) .unwrap_or_default(); - if should_run_ad_stack && is_html_content_type(origin_content_type) { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); - // Every CDN-targeted cache directive, not just the browser-facing - // `Cache-Control` above: an origin emitting any of these would otherwise - // instruct an intermediary to store a synthesized per-navigation - // document. `Surrogate-Control` and `Fastly-Surrogate-Control` cover - // Fastly; `CDN-Cache-Control` is the standard targeted field (RFC 9213) - // and `Cloudflare-CDN-Cache-Control` is the Cloudflare-specific field - // that overrides it there, so both are needed to close the gap on the - // Cloudflare adapter. - for directive in CDN_CACHE_HEADERS { - response.headers_mut().remove(*directive); + if is_html_content_type(origin_content_type) { + if should_run_ad_stack { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + response.headers_mut().remove(header::ETAG); + response.headers_mut().remove(header::LAST_MODIFIED); + // Every CDN-targeted cache directive, not just the browser-facing + // `Cache-Control` above: an origin emitting any of these would otherwise + // instruct an intermediary to store a synthesized per-navigation + // document. `Surrogate-Control` and `Fastly-Surrogate-Control` cover + // Fastly; `CDN-Cache-Control` is the standard targeted field (RFC 9213) + // and `Cloudflare-CDN-Cache-Control` is the Cloudflare-specific field + // that overrides it there, so both are needed to close the gap on the + // Cloudflare adapter. + for directive in CDN_CACHE_HEADERS { + response.headers_mut().remove(*directive); + } + } else { + let origin_cache_control = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .map(str::to_ascii_lowercase); + if !origin_cache_control + .as_deref() + .is_some_and(|value| value.contains("private") || value.contains("no-store")) + { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + } } } @@ -4697,13 +4713,16 @@ mod tests { .expect("should build conditional navigation request") } - fn queue_cacheable_html_response(stub: &StubHttpClient) { + fn queue_html_response_with_cache_control( + stub: &StubHttpClient, + cache_control: &'static str, + ) { stub.push_response_with_headers( 200, b"origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), + ("cache-control", cache_control), ("etag", ORIGIN_ETAG), ("last-modified", ORIGIN_LAST_MODIFIED), ("surrogate-control", "max-age=300"), @@ -4773,7 +4792,7 @@ mod tests { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_cacheable_html_response(&stub); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -4866,11 +4885,11 @@ mod tests { } #[tokio::test] - async fn navigation_without_matched_slots_preserves_origin_cache_policy() { + async fn navigation_without_matched_slots_uses_short_browser_cache_policy() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_cacheable_html_response(&stub); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -4916,7 +4935,7 @@ mod tests { ); for (header_name, expected) in [ - (header::CACHE_CONTROL, "public, max-age=300"), + (header::CACHE_CONTROL, "max-age=60"), (header::ETAG, ORIGIN_ETAG), (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), ( @@ -4947,6 +4966,36 @@ mod tests { } } + #[tokio::test] + async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "origin {cache_control} policy should not be weakened" + ); + } + } + #[tokio::test] async fn eligible_navigation_rejects_unexpected_origin_304() { for content_type in [None, Some("text/html; charset=utf-8")] { From 92e2a78062dc1b57194cbd8873a73f8ef7e5bdec Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 17:17:40 -0500 Subject: [PATCH 163/395] Document dedicated server-side ad template switch --- ...-server-side-ad-templates-cache-control.md | 238 ++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md diff --git a/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md new file mode 100644 index 000000000..f96bdcb28 --- /dev/null +++ b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md @@ -0,0 +1,238 @@ +# Dedicated Server-Side Ad Templates Switch and Cache Policy Plan + +> **For agentic workers:** Implement this plan task-by-task, keeping the dedicated +> template switch separate from the global auction configuration. + +**Goal:** Add an explicit on/off switch for server-side ad templates, while +retaining the browser-facing cache policy from issue #1007: + +- Server-side ad templates active: `Cache-Control: private, no-store`. +- Server-side ad templates inactive: `Cache-Control: max-age=60`, unless the + origin already sends `private` or `no-store`. +- CDN-specific cache headers must not change when templates are inactive. + +**Issue context:** The current cache-policy change uses the runtime +`should_run_ad_stack` gate. That gate is also affected by `[auction].enabled`, +which is not the right configuration boundary for publisher templates. A +browser can call `POST /auction`, and that endpoint is a separate server-run +auction API. The new switch must disable publisher HTML/page-bids template +delivery without disabling that API. + +## Configuration decision + +Add this field to the existing `[creative_opportunities]` section: + +```toml +[creative_opportunities] +enabled = true +``` + +Use `enabled = false` to turn off server-side ad templates while retaining the +slot definitions and keeping direct `POST /auction` behavior available. + +### Compatibility rules + +- The field defaults to `true` when omitted, preserving existing behavior for + deployments that already have `[creative_opportunities]` configured. +- The section remains optional. An absent section continues to mean that the + feature is unavailable. +- Serialize the default `true` value as omitted, matching the existing + rollback-compatibility pattern for newer creative-opportunity fields. An + explicit `false` must remain serialized so the setting is not silently lost. +- `auction.enabled` remains a separate auction/orchestrator setting. Do not use + it as the dedicated template switch and do not thread the new template flag + into `POST /auction`. + +## Current cache behavior to retain + +The existing HTML policy block in `publisher.rs` must remain structurally +consistent with the current issue #952 behavior: + +1. For an eligible request that runs the server-side ad stack and receives HTML: + - Set `Cache-Control: private, no-store`. + - Remove `ETag` and `Last-Modified`. + - Remove `Surrogate-Control`, `Fastly-Surrogate-Control`, `CDN-Cache-Control`, + and `Cloudflare-CDN-Cache-Control`. +2. For HTML where the server-side ad stack does not run, including an explicit + template disable: + - Read the browser-facing `Cache-Control` header. + - If its value contains `private` or `no-store`, case-insensitively, preserve + the origin value exactly. + - Otherwise set exactly `Cache-Control: max-age=60`. + - Leave validators and all CDN-specific cache headers untouched. +3. Preserve the later adapter response-privacy finalization for cookie-bearing + responses; this plan does not refactor that behavior. + +## File map + +### Configuration and compatibility + +- `crates/trusted-server-core/src/creative_opportunities.rs` + - Add `CreativeOpportunitiesConfig::enabled` with a default-true serde + implementation and documentation. + - Add a small accessor if it improves readability, but keep the source of + truth in this config type. + - Update config constructors and serialization tests. +- `crates/trusted-server-core/src/settings.rs` + - Keep `creative_opportunities` parsing and runtime preparation compatible with + the new field. + - Make `creative_opportunity_slots()` return an empty slice when the section + is absent or explicitly disabled, so all adapters receive one consistent + runtime view. + - Add TOML and environment-override coverage for `enabled = false`. +- `crates/trusted-server-core/src/config.rs` + - Extend legacy-schema tests to prove default `enabled = true` is omitted from + serialized blobs and remains readable by older binaries. + - Prove an explicit `enabled = false` is serialized, making rollback failure + loud rather than silently re-enabling templates. +- `trusted-server.example.toml` + - Document `creative_opportunities.enabled` and show how to turn templates off + without deleting slot definitions. +- `docs/guide/configuration.md` + - Add the field to the creative-opportunities reference and document the + environment override: + `TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false`. + - Clarify that this switch controls publisher HTML/page-bids template + delivery, not direct `POST /auction` callers. +- `CHANGELOG.md` + - Add an entry describing the dedicated template switch and cache behavior. + +### Publisher execution and cache policy + +- `crates/trusted-server-core/src/publisher.rs` + - Include the dedicated flag in the initial publisher eligibility decision. + - Do not match, dispatch, or inject server-side ad templates when the flag is + false, even if slots are configured and `[auction].enabled` is true. + - Apply the issue #1007 inactive-HTML cache policy in this state. + - Update skip-reason diagnostics/telemetry so `ad_templates_disabled` is + distinguishable from `auction_disabled`, consent denial, bots, prefetch, and + no matching slots. + - Update `handle_page_bids` so an explicit template disable returns the normal + empty JSON shape (`slots: []`, `bids: {}`) rather than slot definitions. Keep + the current `404` behavior for an absent `[creative_opportunities]` section. + - Extend the existing SSAT cache-policy and eligibility tests. +- `crates/trusted-server-core/src/auction/endpoints.rs` + - Do not gate `POST /auction` on the new template flag. + - Add a regression test or test fixture proving that disabling + `creative_opportunities.enabled` does not suppress a direct auction request + when providers are configured. + - Separately document/verify the existing behavior of `[auction].enabled` for + this endpoint; do not conflate that global setting with the new template + switch. + +### Adapter propagation and browser behavior + +The adapters already pass `Settings::creative_opportunity_slots()` into the +publisher/page-bids handlers. Update and verify these call sites so the central +empty-slice behavior is honored; avoid adding four divergent config checks: + +- `crates/trusted-server-adapter-fastly/src/app.rs` +- `crates/trusted-server-adapter-axum/src/app.rs` +- `crates/trusted-server-adapter-cloudflare/src/app.rs` +- `crates/trusted-server-adapter-spin/src/app.rs` + +No route-level flag is needed if the core `Settings` accessor and handlers are +correct. Add adapter route assertions only where existing fixtures make them +useful. + +The browser runtime already defaults `window.tsjs.adSlots` and +`window.tsjs.bids` to empty values when the edge does not inject templates. If +terminology is updated, adjust these comments/tests without changing runtime +semantics: + +- `crates/trusted-server-js/lib/src/core/index.ts` +- `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Relevant page-bids tests under `crates/trusted-server-js/lib/test/integrations/gpt/` + +## Implementation tasks + +### Task 1: Add and serialize the dedicated setting + +- [ ] Add `enabled: bool` to `CreativeOpportunitiesConfig` with default `true`. +- [ ] Use `skip_serializing_if` so the default value does not appear in stored + config blobs; explicit `false` must serialize. +- [ ] Update all Rust struct literals in `creative_opportunities.rs` and + `publisher.rs` tests. +- [ ] Add parsing, default, false-value, and environment-override tests. +- [ ] Update the legacy compatibility tests in `config.rs`. + +### Task 2: Thread the setting through publisher eligibility + +- [ ] Update `should_run_server_side_ad_stack` to accept the dedicated template + flag as an explicit gate, with a descriptive parameter/doc comment. +- [ ] Ensure initial publisher slot matching and `Settings::creative_opportunity_slots` + do not expose slots when templates are disabled. +- [ ] Preserve the existing `[auction].enabled` and consent gates as separate + conditions. +- [ ] Add an `ad_templates_disabled` diagnostic/telemetry skip reason where the + current branch records a skipped auction. + +### Task 3: Apply the cache policy to the dedicated-off state + +- [ ] Keep the active-SSAT `private, no-store` behavior and validator/CDN header + removal unchanged. +- [ ] Keep the inactive-HTML `max-age=60` behavior from issue #1007. +- [ ] Verify that explicit template disable changes only browser-facing + `Cache-Control` for cacheable HTML; preserve `ETag`, `Last-Modified`, and + every CDN-specific header. +- [ ] Verify that origin `private`, `PRIVATE`, `no-store`, and `No-Store` values + remain unchanged. + +### Task 4: Gate SPA page-bids/template delivery + +- [ ] Include `co_config.enabled` in the `ad_stack_enabled` decision in + `handle_page_bids`. +- [ ] Return empty slots and bids for an explicit disable while retaining the + endpoint and its existing response privacy headers. +- [ ] Keep the absent-section `404` behavior unchanged. +- [ ] Add tests for enabled, disabled, absent, consent-denied, bot, and prefetch + cases as appropriate; preserve existing tests for `[auction].enabled=false`. + +### Task 5: Protect direct `POST /auction` from accidental coupling + +- [ ] Add a focused endpoint test with `creative_opportunities.enabled=false` + and a recording provider. +- [ ] Assert that the provider still sees the direct auction request and that + the response remains a normal OpenRTB response. +- [ ] If the test reveals that `[auction].enabled=false` also needs a separate + product decision for `/auction`, record that as a follow-up rather than + changing it as part of the template-switch work. + +### Task 6: Update docs, examples, comments, and adapter coverage + +- [ ] Update the example config, configuration guide, and changelog. +- [ ] Update stale comments that call `[auction].enabled` the universal template + kill switch. +- [ ] Verify all four adapter call sites use the centralized disabled-slot view. +- [ ] Run JS tests if comments or tests are touched; no JS behavior change is + expected. + +## Test plan + +Use target-matched commands; do not run bare workspace tests because the +workspace contains multiple runtime targets. + +- [ ] `cargo test-axum -p trusted-server-core publisher` +- [ ] `cargo test-fastly` +- [ ] `cargo test-axum` +- [ ] `cargo test-cloudflare` +- [ ] `cargo test-spin` +- [ ] `cargo fmt --all -- --check` +- [ ] `cargo clippy-fastly` +- [ ] `cargo clippy-axum` +- [ ] `cargo clippy-cloudflare` +- [ ] `cargo clippy-cloudflare-wasm` +- [ ] `cargo clippy-spin-native` +- [ ] `cargo clippy-spin-wasm` +- [ ] `cd crates/trusted-server-js/lib && npx vitest run` if JS tests/comments change +- [ ] `cd docs && npm run format` if documentation formatting is required + +## Non-goals + +- Do not change CDN-specific cache policy for inactive templates. +- Do not change adapter response privacy or cookie handling. +- Do not use `auction.rewrite_creatives` as the template switch; it controls + creative URL rewriting, not whether the server-side template stack runs. +- Do not gate or disable direct `POST /auction` as part of this feature. +- Do not remove slot definitions when the switch is off; the point of the switch + is to provide a reversible runtime control. From 58054463a16ce801198f877b687b579a33a9d9a3 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 18:03:25 -0500 Subject: [PATCH 164/395] Add dedicated server-side ad template switch --- CHANGELOG.md | 1 + .../src/auction/endpoints.rs | 127 +++++++++- crates/trusted-server-core/src/config.rs | 27 ++ .../src/creative_opportunities.rs | 47 +++- crates/trusted-server-core/src/publisher.rs | 237 +++++++++++++++--- crates/trusted-server-core/src/settings.rs | 46 +++- .../trusted-server-js/lib/src/core/index.ts | 8 +- .../lib/src/integrations/gpt/index.ts | 12 +- .../test/integrations/gpt/spa_hook.test.ts | 6 +- docs/guide/configuration.md | 20 +- trusted-server.example.toml | 3 + 11 files changed, 480 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36655763c..7c43a6fa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Publisher HTML now uses `Cache-Control: max-age=60` when server-side ad templates are inactive, while preserving origin `private`/`no-store` policies and CDN-specific cache headers. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers. - **Breaking** — `bid_param_zone_overrides` inner values must now be JSON objects; previously non-object or empty values (`"header" = "x"`, `"header" = {}`) were accepted and silently produced a dead rule at runtime. They now fail at startup with a configuration error. Operators upgrading should audit their `bid_param_zone_overrides` config for non-object zone entries. - **Breaking** — Integration configuration strings are no longer globally reinterpreted as JSON scalars. Operators upgrading should audit `[integrations.*]` settings and use native TOML/typed-config booleans and numbers (for example, `enabled = true`, not `enabled = "true"`); quoted numeric and boolean scalars now fail validation instead of silently converting. - **Breaking** — Sourcepoint browser module inclusion now requires explicit `[integrations.sourcepoint].enabled = true`; operators relying on the previous unconditional Sourcepoint module should enable the integration before upgrading. diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 36e083948..b80b9339c 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -554,10 +554,14 @@ mod tests { use crate::consent::types::ConsentContext; use crate::openrtb::Uid; use crate::platform::test_support::{ - NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, noop_services, + NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, StubHttpClient, + noop_services, }; - use crate::platform::{ClientInfo, PlatformPendingRequest, PlatformResponse}; - use crate::test_support::tests::create_test_settings; + use crate::platform::{ + ClientInfo, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, + PlatformResponse, + }; + use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use serde_json::json; @@ -642,6 +646,123 @@ mod tests { } } + /// Provider used to prove that direct `/auction` remains available when + /// publisher server-side ad templates are disabled. + struct TemplateSwitchProbeProvider { + calls: Arc>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for TemplateSwitchProbeProvider { + fn provider_name(&self) -> &'static str { + "template_switch_probe" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + *self.calls.lock().expect("should lock provider call count") += 1; + let request = Request::builder() + .method("POST") + .uri("https://bidder.example/auction") + .body(EdgeBody::empty()) + .expect("should build probe provider request"); + context + .services + .http_client() + .send_async(PlatformHttpRequest::new( + request, + "template-switch-probe-backend", + )) + .await + .change_context(TrustedServerError::Auction { + message: "probe provider launch failed".to_string(), + }) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::success( + self.provider_name(), + Vec::new(), + 0, + )) + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("template-switch-probe-backend".to_string()) + } + } + + #[tokio::test] + async fn direct_auction_remains_available_when_templates_are_disabled() { + let settings_toml = format!( + "{}\n[auction]\nenabled = true\nproviders = [\"template_switch_probe\"]\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&settings_toml) + .expect("should parse settings with disabled templates"); + let calls = Arc::new(Mutex::new(0)); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(TemplateSwitchProbeProvider { + calls: Arc::clone(&calls), + })); + + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"probe response".to_vec()); + let services = RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::clone(&stub) as Arc) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .build(); + let ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let body = json!({ + "adUnits": [{ + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + }); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize body"), + )) + .expect("should build auction request"); + + let response = handle_auction( + &settings, + &orchestrator, + None, + None, + &ec_context, + &services, + req, + ) + .await + .expect("direct auction should remain available"); + + assert_eq!( + *calls.lock().expect("should lock provider call count"), + 1, + "disabling publisher templates must not disable direct /auction" + ); + assert_eq!(response.status(), StatusCode::OK); + } + #[tokio::test] async fn auction_endpoint_consent_gate_returns_no_bid_without_contacting_providers() { // GDPR/unknown jurisdiction lacking effective TCF Purpose 1 must not run diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index cdee3b222..033e6c908 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -321,10 +321,37 @@ formats = [{ width = 300, height = 250 }] fn absent_gam_unit_template_is_accepted_by_legacy_schema() { let creative_opportunities = serialized_creative_opportunities(None); + assert!( + creative_opportunities.get("enabled").is_none(), + "default template switch should be omitted for legacy binaries" + ); serde_json::from_value::(creative_opportunities) .expect("should accept absent GAM unit template"); } + #[test] + fn disabled_creative_opportunities_flag_is_visible_to_legacy_schema() { + let mut toml = crate_test_settings_str(); + toml.push_str( + r#" + +[creative_opportunities] +enabled = false +gam_network_id = "99999" +"#, + ); + let app_config: TrustedServerAppConfig = + toml::from_str(&toml).expect("should deserialize app config wrapper"); + let creative_opportunities = serde_json::to_value(app_config) + .expect("should serialize app config wrapper") + .get("creative_opportunities") + .cloned() + .expect("should contain creative opportunities"); + + serde_json::from_value::(creative_opportunities) + .expect_err("legacy binaries should reject an explicit disabled switch"); + } + #[test] fn deploy_validation_rejects_placeholders() { let settings = Settings::from_toml( diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 10a85b3e8..a6b70e1d6 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -183,10 +183,27 @@ fn derive_section(path: &str, section_root: &str, section_segment: usize) -> Str } } +const fn default_enabled() -> bool { + true +} + +const fn is_default_enabled(value: &bool) -> bool { + *value == default_enabled() +} + /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct CreativeOpportunitiesConfig { + /// Enables server-side ad template delivery on publisher HTML and page-bids requests. + /// + /// This does not disable the direct `POST /auction` endpoint. The default is + /// `true` so existing creative-opportunity configurations retain their behavior. + #[serde( + default = "default_enabled", + skip_serializing_if = "is_default_enabled" + )] + pub enabled: bool, /// GAM network ID used to build default unit paths. pub gam_network_id: String, /// Maximum time in milliseconds to wait for the server-side auction before @@ -244,7 +261,7 @@ pub struct CreativeOpportunitiesConfig { /// [`section_root`](Self::section_root) are omitted. #[serde(default, skip_serializing_if = "Option::is_none")] pub section_segment: Option, - /// Slot templates. Empty vec = feature disabled (no auction fired, no globals injected). + /// Slot templates. An empty vec or `enabled = false` disables template delivery. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub slot: Vec, } @@ -1139,12 +1156,39 @@ mod tests { assert_eq!(derive_section("/%%%/x", "home", 0), "_"); } + #[test] + fn enabled_defaults_true_and_is_omitted_from_serialized_config() { + let config = make_config_with_section_template(None); + assert!( + config.enabled, + "template delivery should default to enabled" + ); + let value = serde_json::to_value(&config).expect("should serialize config"); + assert!( + value.get("enabled").is_none(), + "default enabled value should be omitted for rollback compatibility" + ); + } + + #[test] + fn disabled_template_switch_is_serialized() { + let mut config = make_config_with_section_template(None); + config.enabled = false; + let value = serde_json::to_value(&config).expect("should serialize config"); + assert_eq!( + value.get("enabled"), + Some(&serde_json::Value::Bool(false)), + "explicitly disabled template delivery must remain in config blobs" + ); + } + fn make_config_with_section_template( section_root: Option<&str>, ) -> CreativeOpportunitiesConfig { let mut slot = make_slot("ad-header-0", vec!["/news/*"]); slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "99999".to_string(), auction_timeout_ms: None, price_granularity: PriceGranularity::default(), @@ -1542,6 +1586,7 @@ mod tests { // Older binaries deserialize this struct with `deny_unknown_fields`, so // a pushed config blob must not carry `"section_root": null`. let config = CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "99999".to_string(), auction_timeout_ms: None, price_granularity: PriceGranularity::default(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8919f09b3..87ab8317b 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1751,27 +1751,34 @@ pub(crate) fn is_prefetch_request(req: &Request) -> bool { header("sec-purpose") || header("purpose") } -/// Returns true only when the publisher request should run the full -/// server-side ad stack: auction dispatch plus initial ad-slot injection. +#[derive(Debug, Clone, Copy)] +struct ServerSideAdStackConfig { + /// Dedicated `[creative_opportunities].enabled` switch. + ad_templates_enabled: bool, + /// Global `[auction].enabled` gate used by publisher/page-bids flows. + auction_enabled: bool, +} + +/// Returns true only when the publisher should inject and run server-side ad templates. /// -/// `auction_enabled` is the global `[auction].enabled` kill switch — when -/// false, no automatic server-side auction or ad-slot injection runs. -pub(crate) fn should_run_server_side_ad_stack( +/// This includes auction dispatch plus initial ad-slot injection. +fn should_run_server_side_ad_stack( is_get: bool, is_navigation: bool, is_prefetch: bool, is_bot: bool, has_matched_slots: bool, consent_allows_auction: bool, - auction_enabled: bool, + config: ServerSideAdStackConfig, ) -> bool { is_get && is_navigation && !is_prefetch && !is_bot + && config.ad_templates_enabled && has_matched_slots && consent_allows_auction - && auction_enabled + && config.auction_enabled } /// Write winning bids from an auction result into the shared `ad_bids_state` lock. @@ -2632,7 +2639,10 @@ pub async fn handle_publisher_request( let is_prefetch = is_prefetch_request(&req); let is_bot = is_bot_user_agent(&req); - let matched_slots = if is_get { + let creative_opportunities = settings.creative_opportunities.as_ref(); + let ad_templates_enabled = creative_opportunities.is_some_and(|co_config| co_config.enabled); + let ad_templates_disabled = creative_opportunities.is_some_and(|co_config| !co_config.enabled); + let matched_slots = if is_get && ad_templates_enabled { settings .creative_opportunities .as_ref() @@ -2655,7 +2665,10 @@ pub async fn handle_publisher_request( is_bot, !matched_slots.is_empty(), consent_allows_auction, - auction.orchestrator.is_enabled(), + ServerSideAdStackConfig { + ad_templates_enabled, + auction_enabled: auction.orchestrator.is_enabled(), + }, ); let should_run_auction = should_run_ad_stack; // Diagnostic: shows which gate suppresses the server-side auction. Pair with @@ -2663,14 +2676,16 @@ pub async fn handle_publisher_request( // when `consent_allows_auction=false`. log::debug!( "server-side ad-stack gate: is_get={is_get} is_navigation={is_navigation} \ - is_prefetch={is_prefetch} is_bot={is_bot} matched_slots={} \ - consent_allows_auction={consent_allows_auction} orchestrator_enabled={} \ - -> should_run_auction={should_run_auction}", + is_prefetch={is_prefetch} is_bot={is_bot} ad_templates_enabled={ad_templates_enabled} \ + matched_slots={} consent_allows_auction={consent_allows_auction} \ + orchestrator_enabled={} -> should_run_auction={should_run_auction}", matched_slots.len(), auction.orchestrator.is_enabled(), ); - if matched_slots.is_empty() && settings.creative_opportunities.is_some() { + if ad_templates_disabled { + log::debug!("Server-side ad templates are disabled by configuration"); + } else if matched_slots.is_empty() && settings.creative_opportunities.is_some() { log::debug!( "No creative opportunity slots matched path '{}' — skipping auction and injection", request_path @@ -2795,7 +2810,9 @@ pub async fn handle_publisher_request( } } } else { - let skip_reason = if !auction.orchestrator.is_enabled() { + let skip_reason = if ad_templates_disabled { + "ad_templates_disabled" + } else if !auction.orchestrator.is_enabled() { "auction_disabled" } else if !consent_allows_auction { "consent_denied" @@ -3771,7 +3788,11 @@ pub async fn handle_page_bids( }) .unwrap_or_else(|| "/".to_string()); - let matched_slots = match_renderable_slots(auction.slots, co_config, &path_param); + let matched_slots = if co_config.enabled { + match_renderable_slots(auction.slots, co_config, &path_param) + } else { + Vec::new() + }; let request_info = crate::http_util::RequestInfo::from_request(&req, services.client_info()); let ec_id = ec_context.ec_value().filter(|_| ec_context.ec_allowed()); @@ -3790,7 +3811,10 @@ pub async fn handle_page_bids( let is_bot = is_bot_user_agent(&req); let auction_enabled = auction.orchestrator.is_enabled(); - if !auction_enabled { + let ad_templates_enabled = co_config.enabled; + if !ad_templates_enabled { + log::debug!("page-bids: [creative_opportunities].enabled is false — skipping templates"); + } else if !auction_enabled { log::debug!("page-bids: [auction].enabled is false — skipping auction"); } else if matched_slots.is_empty() { log::debug!( @@ -3806,14 +3830,14 @@ pub async fn handle_page_bids( ); } - // The [auction].enabled kill switch and a consent denial disable the entire - // server-side ad stack. In those states the endpoint must return no slots, - // so the SPA hook does not assign `ts.adSlots` and call `adInit()` — - // otherwise the kill switch/consent gate would stop SSP calls but still let - // the client create/refresh GPT slots. Bot/prefetch requests, by contrast, - // keep their slot definitions (the placement structure is unchanged) but - // skip the live auction, matching the existing bot/prefetch behaviour. - let ad_stack_enabled = auction_enabled && consent_allows_auction; + // The dedicated template switch, [auction].enabled, and a consent denial + // disable the entire server-side ad stack. In those states the endpoint must + // return no slots, so the SPA hook does not assign `ts.adSlots` and call + // `adInit()` — otherwise the gate would stop SSP calls but still let the + // client create/refresh GPT slots client-side. Bot/prefetch requests, by + // contrast, keep their slot definitions (the placement structure is + // unchanged) but skip the live auction, matching the existing behavior. + let ad_stack_enabled = ad_templates_enabled && auction_enabled && consent_allows_auction; let winning_bids = if matched_slots.is_empty() { std::collections::HashMap::new() @@ -3903,7 +3927,9 @@ pub async fn handle_page_bids( } } } else { - let skip_reason = if !auction_enabled { + let skip_reason = if !ad_templates_enabled { + "ad_templates_disabled" + } else if !auction_enabled { "auction_disabled" } else if !consent_allows_auction { "consent_denied" @@ -4655,6 +4681,15 @@ mod tests { .expect("should parse settings with auction and creative opportunities enabled") } + fn settings_with_disabled_ad_templates() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n\ + [creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled ad templates") + } + fn settings_with_dispatching_provider() -> Settings { let toml = format!( "{}\n[auction]\nenabled = true\nproviders = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n\ @@ -4966,6 +5001,72 @@ mod tests { } } + #[tokio::test] + async fn disabled_ad_templates_use_short_browser_cache_policy() { + // Arrange + let settings = settings_with_disabled_ad_templates(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let response = run_with_slots( + &settings, + &services, + &slots, + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "disabled server-side ad templates should not bypass the origin cache" + ); + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("max-age=60"), + "disabled server-side ad templates should use the short browser cache policy" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "disabled server-side ad templates should preserve {header_name}" + ); + } + } + #[tokio::test] async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { let settings = settings_with_enabled_auction_and_creative_opportunities(); @@ -5450,39 +5551,69 @@ mod tests { #[test] fn server_side_ad_stack_runs_only_when_all_auction_gates_pass() { + let enabled_config = ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: true, + }; assert!( - should_run_server_side_ad_stack(true, true, false, false, true, true, true), - "GET, real navigation, matched slots, and consent should run TS ad stack" + should_run_server_side_ad_stack(true, true, false, false, true, true, enabled_config,), + "GET, real navigation, enabled templates, matched slots, and consent should run TS ad stack" ); assert!( - !should_run_server_side_ad_stack(false, true, false, false, true, true, true), + !should_run_server_side_ad_stack(false, true, false, false, true, true, enabled_config,), "non-GET requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, false, false, false, true, true, true), + !should_run_server_side_ad_stack(true, false, false, false, true, true, enabled_config,), "non-document requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, true, false, true, true, true), + !should_run_server_side_ad_stack(true, true, true, false, true, true, enabled_config,), "prefetch requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, true, true, true, true), + !should_run_server_side_ad_stack(true, true, false, true, true, true, enabled_config,), "bot requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, false, true, true), + !should_run_server_side_ad_stack(true, true, false, false, false, true, enabled_config,), "requests with no matching slots should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, false, true), + !should_run_server_side_ad_stack(true, true, false, false, true, false, enabled_config,), "requests without required consent should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, true, false), + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: false, + }, + ), "disabled [auction].enabled kill switch should skip TS ad stack and injection" ); + assert!( + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: false, + auction_enabled: true, + }, + ), + "disabled [creative_opportunities].enabled switch should skip TS ad stack and injection" + ); } #[tokio::test] @@ -8120,6 +8251,7 @@ mod tests { fn make_config() -> CreativeOpportunitiesConfig { CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "21765378893".to_string(), auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, @@ -9337,6 +9469,14 @@ mod tests { Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } + fn settings_with_co_templates_disabled() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled templates") + } + async fn run_page_bids( settings: &Settings, orchestrator: &AuctionOrchestrator, @@ -9886,6 +10026,35 @@ mod tests { ); } + #[tokio::test] + async fn disabled_server_side_ad_templates_return_no_slots_or_bids() { + // The dedicated template switch must suppress publisher/page-bids + // delivery without using the global auction switch. + let settings = settings_with_co_templates_disabled(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let req = make_page_bids_request("/2024/01/my-article/"); + + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 0, + "disabled server-side ad templates must not return slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "disabled server-side ad templates must not produce bids" + ); + } + #[tokio::test] async fn consent_denied_returns_no_slots_or_bids() { // When consent denies the server-side auction (here: Jurisdiction diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 1021ee9ec..06152659c 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2095,13 +2095,14 @@ impl Settings { Ok(()) } - /// Returns compiled creative opportunity slots, or empty slice if feature is disabled. + /// Returns compiled creative opportunity slots when template delivery is enabled. #[must_use] pub fn creative_opportunity_slots( &self, ) -> &[crate::creative_opportunities::CreativeOpportunitySlot] { self.creative_opportunities .as_ref() + .filter(|co| co.enabled) .map(|co| co.slot.as_slice()) .unwrap_or(&[]) } @@ -5010,6 +5011,10 @@ formats = [{ width = 300, height = 250 }] let co = settings .creative_opportunities .expect("should have creative_opportunities"); + assert!( + co.enabled, + "creative-opportunity templates should default to enabled" + ); assert_eq!(co.gam_network_id, "21765378893"); assert_eq!(co.auction_timeout_ms, Some(500)); assert_eq!( @@ -5019,6 +5024,45 @@ formats = [{ width = 300, height = 250 }] ); } + #[test] + fn settings_disables_creative_opportunity_slots_when_configured_off() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = false\ngam_network_id = \"21765378893\"\n\n[[creative_opportunities.slot]]\nid = \"atf\"\npage_patterns = [\"/\"]\nformats = [{{ width = 300, height = 250 }}]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("should parse disabled templates"); + assert!( + settings.creative_opportunity_slots().is_empty(), + "disabled template delivery should expose no runtime slots" + ); + } + + #[test] + fn settings_creative_opportunity_enabled_flag_supports_environment_override() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = true\ngam_network_id = \"21765378893\"\n", + crate_test_settings_str() + ); + let env_key = format!( + "{}{}CREATIVE_OPPORTUNITIES{}ENABLED", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + + temp_env::with_var(env_key, Some("false"), || { + let settings = Settings::from_toml_and_env(&toml) + .expect("should parse template enabled environment override"); + assert!( + !settings + .creative_opportunities + .expect("should have creative opportunities") + .enabled, + "environment override should disable template delivery" + ); + }); + } + #[test] fn settings_rejects_invalid_creative_opportunity_slot_id() { let toml = r#" diff --git a/crates/trusted-server-js/lib/src/core/index.ts b/crates/trusted-server-js/lib/src/core/index.ts index 5d8e41971..b2c4e41e1 100644 --- a/crates/trusted-server-js/lib/src/core/index.ts +++ b/crates/trusted-server-js/lib/src/core/index.ts @@ -36,10 +36,10 @@ api.getConfig = getConfig; // Provide core requestAds API api.requestAds = requestAds; // Defensive defaults: the edge injects adSlots (head-open) and bids (before -// ) only when the server-side ad stack runs for the request. When it -// is gated off (kill switch, consent fail-closed, bots, prefetch), page code -// reading window.tsjs.bids / window.tsjs.adSlots must still see defined -// values instead of throwing. Injected scripts overwrite these wholesale. +// ) only when server-side ad templates run for the request. When template +// delivery is disabled or gated off (auction/consent, bots, prefetch), page code +// reading window.tsjs.bids / window.tsjs.adSlots must still see defined values +// instead of throwing. Injected scripts overwrite these wholesale. api.adSlots ??= []; api.bids ??= {}; // Point global tsjs diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 30cfbbbdd..acea49fa6 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -689,8 +689,8 @@ export function installTsAdInit(): void { ts.prevSlotTargetingKeys = nextSlotTargetingKeys; // Whether this call produced any TS slot to render. A gated page-bids - // response (auction kill switch or consent denial) returns no slots, so - // the loops above leave these empty. + // response (template switch, auction gate, or consent denial) returns no + // slots, so the loops above leave these empty. const hasRenderableWork = slotsToDisplay.length > 0 || slotsToRefresh.length > 0; // enableSingleRequest and enableServices must only be called once per page @@ -971,10 +971,10 @@ export function installSpaAuctionHook(): void { // This route is now the committed, loaded state — a later failed // navigation rolls back here, and a return trip no-ops correctly. lastAppliedPath = path; - // An empty page-bids response (auction kill switch or consent gate) carries - // no TS slots. Only run adInit() when there are slots to apply or prior TS - // state to sweep — otherwise a consent-denied or kill-switched navigation - // must not enter the GPT command queue and risk activating services. + // An empty page-bids response (template switch, auction, or consent gate) + // carries no TS slots. Only run adInit() when there are slots to apply or + // prior TS state to sweep — otherwise a gated navigation must not enter + // the GPT command queue and risk activating services. const hasPriorTsState = (ts.prevGptSlots?.length ?? 0) > 0 || Object.keys(ts.prevSlotTargetingKeys ?? {}).length > 0 || diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 19739e3dd..36c41aa71 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -115,9 +115,9 @@ describe('installSpaAuctionHook', () => { }); it('skips adInit on an empty page-bids response with no prior TS state', async () => { - // A gated page-bids response (auction kill switch or consent denial) returns - // no slots. With no prior TS state to sweep, the hook must not call adInit() - // so a consent-denied navigation cannot activate the publisher's GPT setup. + // A gated page-bids response (template switch, auction gate, or consent + // denial) returns no slots. With no prior TS state to sweep, the hook must + // not call adInit() so a gated navigation cannot activate publisher GPT. fetchStub.mockResolvedValue({ ok: true, json: async () => ({ slots: [], bids: {} }), diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index aa2b0264b..ef991eea0 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1312,8 +1312,16 @@ Defines the ad slots the trusted server offers on a page: which pages each slot appears on (`page_patterns`), its supported sizes (`formats`), and the GAM ad unit it maps to (`gam_unit_path`). +`enabled` is the dedicated server-side ad-template switch. It defaults to `true` +for compatibility with existing configurations. Set it to `false` to stop +publisher HTML and SPA page-bids template delivery while retaining the slot +configuration and direct `POST /auction` endpoint. The browser-facing cache +policy for a disabled template stack is `Cache-Control: max-age=60`, unless the +origin already sends `private` or `no-store`. + ```toml [creative_opportunities] +enabled = true # set to false to disable server-side ad templates gam_network_id = "123456789" price_granularity = "dense" @@ -1332,6 +1340,13 @@ page_patterns = ["/", "/news", "/news/*", "/reviews", "/reviews/*"] formats = [{ width = 728, height = 90 }] ``` +The same switch can be overridden through the legacy environment-variable +loader: + +```bash +TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false +``` + ### `gam_unit_path` templating `gam_unit_path` is a template. A publisher whose ad unit varies by site section @@ -1385,8 +1400,9 @@ publisher-specific. Startup fails if `{section}` is used without a valid `section_root`. Startup rejects a blank `gam_network_id` only when an absent path/default or a `{network_id}` template consumes it; static paths and templates without `{network_id}` do not consume it. A -`[creative_opportunities]` block with no slots is disabled, so its -`gam_network_id` is not checked. +`[creative_opportunities]` block with `enabled = false` or no slots is +inactive, so no publisher templates are delivered and its `gam_network_id` is +not checked when no slot uses it. Both knobs are config-driven, so the URL→section convention stays with the publisher: `section_segment` selects which segment names the section, and diff --git a/trusted-server.example.toml b/trusted-server.example.toml index e78d2b255..ef5bf5487 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -155,6 +155,9 @@ ja4_endpoint_enabled = false auction_html_comment = false [creative_opportunities] +# Set to false to disable server-side ad templates while retaining slot definitions +# and direct POST /auction callers. +enabled = true gam_network_id = "123456789" # FCP is not affected by this value — body content above has already # streamed and painted before the hold begins. What this caps is the slip on From 0476999aa048ef63ac1ab333ee8ca4902f22ad9e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 8 Aug 2026 12:23:25 +0530 Subject: [PATCH 165/395] Add ESI cacheable-root validation design for #1009 Validates the ESI approach proposed in #1009 and recommends deferring it. ESI presupposes a TS-owned template cache: its pull-based BufRead input cannot sit downstream of lol_html's push-based rewriter without an intermediate buffer, and the cache boundary is that buffer. That cache is in turn blocked on purge capability the service does not have. Revival condition: React #418 resolved and the window.load gate removed. Re-diagnoses the TTFB regression the issue targets. The auction is dispatched before the origin fetch and does not block, and on a Next.js publisher the closing body tag is not reached until the whole document has been buffered, so the auction hold costs approximately nothing. The cost is with_cache_bypass forcing a readthrough-cache miss on every ad-eligible navigation. Removing either alone recovers little; the two are multiplicative. Corrects nine premises in the issue, including that tsjs.adSlots is per-URL rather than per-user, and that moving identity off the inline response is a prerequisite only for a visitor's first navigation. Carries no performance measurements. Every conclusion is derived from code at the pinned baseline so it can be checked by reading the repository. --- ...08-esi-cacheable-root-validation-design.md | 635 ++++++++++++++++++ 1 file changed, 635 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md new file mode 100644 index 000000000..4e456e2d0 --- /dev/null +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -0,0 +1,635 @@ +# ESI and the Cacheable Root + +**Issue:** IABTechLab/trusted-server#1009 · **Date:** 2026-08-08 +**Baseline:** citations verified at `cfb98f4`; unchanged as of `b0ce56c3` (the two +commits between touch only CI workflows and Cargo aliases). + +**Decision requested:** approve the four items below. Three are "yes/no"; one funds +about three days of measurement. + +> **Orientation.** #1009 asks whether Edge Side Includes (ESI) can cache page fragments +> so that cacheable publisher HTML is separated from per-user ad state, recovering a +> TTFB regression that Trusted Server (TS) adds to navigations on a Next.js App Router +> publisher running on Fastly Compute. The answer is no to ESI, and the regression has a +> cheaper cause than the issue assumes. +> +> **This document deliberately carries no performance measurements.** Every conclusion +> below is derived from code at the pinned baseline, so it can be checked by reading the +> repository rather than by trusting a benchmark. Where a quantity is needed and unknown, +> it is named as unknown and [§3](#3-monday-morning) says how to obtain it. +> +> Terms used throughout: **the hold** = TS holding the HTTP response open at `` +> until the server-side auction (SSAT) resolves. **#418** = a React hydration-mismatch +> defect caused by `adInit()` mutating ad-slot subtrees during hydration; it is why bid +> application is deferred to `window.load`. **The SSAT price defect** = a live +> mispricing bug named in #1009 (prices reading 100× high) — cited from #1009 and prior +> investigation, not re-verified here. + +--- + +## 1. Decision requested + +| # | Decision | Owner needed | +| --- | -------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| D1 | **ESI is deferred.** Revival condition: #418 resolved _and_ the `window.load` gate removed. Not a rejection — a dated condition. | Eng + product | +| D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | +| D3 | **Approve Stage 0** (remove `with_cache_bypass`), subject to the origin-`Vary` check in §3. | Eng | +| D4 | **Stages 1–2 queue behind the SSAT price defect and #418.** Stages 3b–5 unscheduled. | Product | + +Rationale for D4 in [§8](#8-priority). Everything this document recommends _against_ +doing is in [§7](#7-deferred-work-specified-not-scheduled), at deliberately lower +detail than the work it recommends. + +--- + +## 2. Why — the three findings + +**ESI does not work here, for a structural reason #1009 misses.** ESI's input is pull +(`BufRead`); `lol_html`'s is push (`HtmlRewriter::write`). ESI cannot sit downstream of +the rewriter without an intermediate buffer, and in the two-stage design the cache +boundary _is_ that buffer. **ESI presupposes a TS-owned template cache** rather than +being independent of one — and that cache is blocked on purge capability TS does not +have (no `Surrogate-Key` anywhere; the Fastly management token is scoped without purge +permission). ESI is also Fastly-only at every API level. Its one advantage over a +client fetch — no round trip — is worth nothing while bids are not consumed until +`window.load`. Separately, enabling ESI's Dynamic Content Assembly would be an SSRF +vector: bid payloads carry partner-controlled creative markup, so an SSP could embed +`` and make the edge fetch an arbitrary URL. Details in +[Appendix E](#appendix-e--esi-implementation-notes). + +**The auction is already out of band; the hold is ~free.** It is dispatched _before_ +the origin fetch and does not block ([publisher.rs:2698-2701](../../../crates/trusted-server-core/src/publisher.rs#L2698-L2701)), +with a 500 ms budget. The actual cost is `with_cache_bypass` +([publisher.rs:2867](../../../crates/trusted-server-core/src/publisher.rs#L2867)), +which forces every ad-eligible navigation to miss the Fastly readthrough cache. + +**The two fixes are multiplicative.** Removing the bypass alone lets the previously +hidden auction surface as the new bottleneck. Removing the hold alone changes nothing, +because the auction was never the bottleneck. **Shipping the hold removal without the +bypass removal will measure no improvement and will read as the effort having failed** — +the most likely way this work gets judged unfairly. + +**Ordering is established; magnitude is not.** The ordering above follows from code and +needs no measurement. The _size_ of the win does — and the one quantity it depends on, +the origin build time under `Pass`, has never been measured. #1009's timings do not +supply it: they compare cached fetches against each other, not against an origin build. +**Quote no figure to a publisher until §3 Step C runs.** Full reasoning in +[§6](#6-the-analysis). + +--- + +## 3. Monday morning + +Three checks, ordered cheapest-first. Each needs a named owner before starting. + +**Step A — origin `Vary` check (minutes).** `curl` the origin with and without `RSC`, +`Next-Router-*`, and the experiment header; inspect the `Vary` response header. +**Gates Stage 0**, the only build item recommended now. Do this first because it is the +cheapest thing that unblocks anything. + +**Step B — what consumes TS's own response headers (under a day).** Request a TS-served +path that already emits `public, s-maxage` +([http_util.rs:294-311](../../../crates/trusted-server-core/src/http_util.rs#L294-L311)) +twice and look for `x-cache`/`age` on TS's _own_ response. **Gates the Stage 3a/3b +split** — see [§7](#7-deferred-work-specified-not-scheduled). + +**Step C — server-side latency breakdown (1 day + a measurement window).** Emit four +timings per ad-eligible navigation: origin fetch duration (this is `O`, the quantity +the model lacks), auction collect duration, rewrite duration, total. Capture with the +bypass both on and off. + +- **Mechanism: `Server-Timing`.** Chosen, not offered — it needs no new plumbing and is + readable from the same browser harness that produced #1009's numbers. +- **Sample: enough navigations per arm to separate the medians with confidence**, across + both page types, and state the N alongside any result. #1009's sample was small enough + that its conclusion did not survive contact with the code; replacing it with another + underpowered sample would repeat the error. + +**Step C has two outcomes, both actionable:** + +| Outcome | Meaning | Effect on staging | +| -------------------------- | --------------------- | ------------------------------------------------------------ | +| `O` materially exceeds `A` | The model in §6 holds | Proceed as staged: Stage 0 primary, Stage 2 protects its win | +| `A` exceeds `O` | The hold _is_ costing | **Staging inverts** — Stage 2 primary, Stage 0 secondary | + +The work does not change; its order and justification do. **The staging in §7 is +conditional on this measurement.** + +Step C also yields the client fetch latency that sets Stage 1's bids timeout, replacing +an invented constant. + +--- + +## 4. Stage 0 — the only build item recommended now + +Remove `with_cache_bypass` at [publisher.rs:2867](../../../crates/trusted-server-core/src/publisher.rs#L2867). + +**Why it is safe in principle.** The conditional-header strip runs 34 lines earlier +under the same gate ([publisher.rs:2832-2836](../../../crates/trusted-server-core/src/publisher.rs#L2832), +which also strips `Range`/`If-Range`), so the request already reaches the cache +unconditional and a HIT returns a full body. [The 304-prevention design](./2026-07-22-ssat-root-document-304-prevention-design.md) +added the bypass as belt-and-braces and listed the TTFB cost under its own Risks. The +strip alone satisfies its invariant. + +**But it carries a risk that design never considered — and this is the blocking +precondition.** RSC fetches are not navigations +([is_navigation_request](../../../crates/trusted-server-core/src/http_util.rs#L73-L98) +requires `Sec-Fetch-Dest: document`), so they never set the bypass and **already flow +through the readthrough cache**, while HTML navigations are `PASS`. Removing the bypass +puts both representations under one cache key. #1009 states the origin varies on +`rsc`, `next-router-*`, and a publisher-specific experiment header — if that variance is +not declared via `Vary`, the +cache can serve a flight payload to an HTML navigation. + +The classification is also not airtight: `is_navigation_request` falls back to the +`Accept` header when Fetch Metadata is absent, and its own comment warns _"this path is +weaker — `fetch()` can set Accept: text/html"_ +([http_util.rs:84-88](../../../crates/trusted-server-core/src/http_util.rs#L84)). + +**Two effort branches, and Step A decides which:** + +| Step A result | Stage 0 is… | Effort | +| ---------------------- | --------------------------------------------- | ------ | +| Origin declares `Vary` | a one-line deletion plus test updates | 1–2 d | +| Origin does **not** | a TS-side cache-key discriminator — a feature | 4–8 d | + +The discriminator is the safer design either way, because it keys on the headers that +actually distinguish the representations rather than on the navigation classification. + +**Two benefits beyond TTFB, worth stating to a publisher:** + +- **Origin load drops.** The 304-prevention design explicitly accepted _"increasing + origin load"_ as a cost. This reverses it. +- **`stale-if-error` becomes reachable.** Under `Pass` an origin outage is a hard + failure. This needs a decision rather than a default: stale HTML carries stale slot + markup, and whether that beats an error is a product call. + +--- + +## 5. The trap in the deferred work — read this before scheduling Stages 1–2 + +The hold is load-bearing for something other than latency. The invariant is: + +> `ad_bids_state` must be `Some(..)` when `lol_html` processes the `` end tag. + +The end-tag handler ([html_processor.rs:381-395](../../../crates/trusted-server-core/src/html_processor.rs#L381-L395)) +locks that mutex once and falls back to `build_empty_bids_script()` on `None`. + +**Removing the hold without relocating collection renders a normal page with +`tsjs.bids = {}` and no server-side ads** — no error, no non-2xx, no ERROR log. On +Axum, Cloudflare, and Spin the loss is fully silent: +[publisher.rs:2248](../../../crates/trusted-server-core/src/publisher.rs#L2248) holds a +bare `Option` with no guard, so not even a drop warning fires. **The +SSPs are billed regardless.** + +This is why Stage 2 is gated on three companions and a production soak, and why slot +fill cannot be the canary — see [§7](#7-deferred-work-specified-not-scheduled). + +--- + +## 6. The analysis + +### 6.1 Corrections to #1009's premises + +| # | #1009 states | Verified against `cfb98f4` | +| --- | -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Two per-user injection seams | `tsjs.adSlots` is per-URL — [build_slot_json](../../../crates/trusted-server-core/src/publisher.rs#L3501-L3525) emits config- and path-derived fields only. **One per-user hole.** | +| 2 | Identity off-inline is a prerequisite | Privacy net is cookie-gated and returning navs set no cookie ([ec/finalize.rs:86-94](../../../crates/trusted-server-core/src/ec/finalize.rs#L86-L94)). **First-visit only.** | +| 3 | Stamp at `:2882-2888` | [`:2945-2963`](../../../crates/trusted-server-core/src/publisher.rs#L2945-L2963), `private, no-store`, also removing `ETag`/`Last-Modified`/four CDN headers. **Seven headers.** | +| 4 | Three cacheability killers | Two more: `bypass_cache` and the [304→502 guard](../../../crates/trusted-server-core/src/publisher.rs#L2894-L2916). **The bypass is the cost.** | +| 5 | Two `!Send` pipelines is the hard part | `?Send` already pervasive. **Not the obstacle.** | +| 6 | Goal: root as a shared Fastly HIT | Nothing caches TS's own response on Compute; the A/B's `x-cache` is the **backend readthrough** cache. **Reframes the goal.** | + +Rows on #1009's `esi` compatibility check (holds), its drifted line numbers, and its +two broken `#1`/`#3` cross-references are in [Appendix A](#appendix-a--full-1009-correction-table). + +**Credit where due.** #1009 names the hold as blocker 1 and states it correctly. What +changes here is its _causal weight_. Likewise, #1009's own observation that TS _"shifts +the auction cost from client-side to server-side rather than adding new work"_ is the +argument for client-fill, which the issue then declines in favour of ESI. + +### 6.2 Why the hold is free — the strong form first + +On a Next.js publisher, `lol_html` never sees the `` end tag until the **final** +chunk: with any post-processor registered, `HtmlWithPostProcessing` accumulates and +emits nothing before then ([html_processor.rs:62-65](../../../crates/trusted-server-core/src/html_processor.rs#L62)), +and the Next.js integration always registers one when enabled +([nextjs/mod.rs:107](../../../crates/trusted-server-core/src/integrations/nextjs/mod.rs#L107)). + +So the auction has the _entire origin download plus rewrite_ to finish before the hold +can block on anything. **The hold cannot cost anything unless the auction outlives the +whole document fetch.** The auction is bounded by the configured `auction_timeout_ms` +([settings.rs:5000](../../../crates/trusted-server-core/src/settings.rs#L5000)), so this +reduces to a single comparison an operator can check against their own config: is the +auction budget larger than a full document fetch and rewrite? If not, the hold is free. + +**This argument uses no timing data at all** — only the code path and one config value. + +The weaker, general form, for publishers with no post-processor registered: because +dispatch precedes the origin fetch, the hold costs `max(0, A − O)`, which is zero +whenever the origin build `O` exceeds the auction budget `A`. + +### 6.3 The quantity nobody has measured + +Write the origin build time under `Pass` as `O`. Recovery depends on it, and it has +never been captured. #1009's timings cannot supply it: they compare a POP hit against a +shield-served fetch, both of which are _cached_ paths, whereas `CacheOverride::Pass` +bypasses every layer and reaches the true origin. The two are different quantities. + +What follows from code alone, without any number: + +| Configuration | Long pole after the change | Recovery | +| ------------------- | -------------------------- | ------------------------------ | +| Hold removal only | origin (still `PASS`) | **none** | +| Bypass removal only | the auction budget | partial — the auction surfaces | +| **Both** | the rewrite | **the full available win** | + +That ordering is what the staging rests on, and it is measurement-independent. The +magnitude of each row is not, and §3 Step C supplies it. + +### 6.4 The ceiling + +#1009 targets "approach the TS-off warm numbers." **Unreachable, structurally.** Those +numbers are TS-off _streaming_ a POP HIT. TS buffers the whole document before emitting +a byte (16 MB cap), so its floor is `full origin body download + full rewrite` — above a +streamed hit by construction, whatever the timings turn out to be. Set the target from +Step C's measured rewrite cost rather than from the TS-off baseline. Going below the +floor requires true origin streaming (#849), out of scope. A non-Next.js publisher with +no post-processor takes the streaming path and would see a lower floor. + +### 6.5 Confidence + +**High on the structural claims.** §6.2's argument, the bypass forcing a cache miss, the +ESI push/pull mismatch, the silent-empty-bids failure mode, the geo and purge blockers, +and the fill-canary blindness are all read directly out of the code at `cfb98f4`. Anyone +can check them without running anything. + +**None on magnitude.** `O` is unmeasured and the rewrite cost is unmeasured. This +document does not estimate them, and no figure in it should be quoted as one. + +Worth stating plainly: #1009 reached the opposite causal conclusion from a small sample. +That is a caution about small samples generally, not only about that one — which is why +§3 Step C specifies the measurement rather than this document supplying a substitute +for it. + +--- + +## 7. Deferred work, specified not scheduled + +Lower detail is deliberate. Full specifications are in the appendices. + +**Stage 1 — bid delivery off the response body.** The client fetches `/_ts/page-bids` at +navigation generation 0. Endpoint, same-origin gate, wire shape, and client consumer +already exist. Three decisions must be made before planning: the `slots: []` precedence +rule when head-open already injected a non-empty `ts.adSlots`; the new terminal-event +emission point; and whether the dispatch/collect split survives at all. Plumbing detail +in [Appendix B](#appendix-b--stage-1-plumbing). Estimated 8–13 d, low-to-medium +confidence, uncertainty concentrated client-side. + +Three companions are mandatory, not optional: **suppress the server bids script +entirely** (not an empty one), **fail loud** (the end-tag handler takes bids by value so +a missing auction is a compile error), and **relocate telemetry** (navigation +`Completed` rows are emitted only from the collect functions, and the `ts-debug` dump +rides the same string). Behaviour change to accept: under client-fill the auction runs +only if the browser executes the fetch, so bots and JS-disabled clients stop triggering +server-side auctions — revenue-relevant, sign unknown. + +**Stage 2 — delete the hold.** 5–8 d. **Rollback is one-way**: it deletes the hold, the +dispatch/collect split, and twelve tests, so the only revert is a release. Ships only +after Stage 1 has run flag-on in production for a window defined _before_ Stage 1 +starts, with TS-attributed renders flat and `auction_events_raw` navigation rows intact. +Secondary wins: removes the duplicated per-codec decoder/encoder wiring, six compression +imports, and the non-parser-context `` carrying the injection point. + +--- + +## Appendix C — `Vary` signal inventory + +Needed for Stage 3b only. + +**Per-user — never shared-cacheable:** consent state (`euconsent-v2`, `__gpp`, +`__gpp_sid`, `us_privacy`, `Sec-GPC`, IP-derived jurisdiction); the GPT-diagnostics +`__Host-ts-console` cookie / `ts_console` query; the `tsjs.bids` payload (removed by +Stage 1, which is what makes the rest tractable); IP-derived geo; DataDome's request +filter, which can replace the document entirely. + +**Per-variant — safe in a cache key:** request host and scheme; `Accept-Encoding`; +request-class headers (`Sec-Fetch-Dest`, `Accept`, `Sec-Purpose`/`Purpose`, bot UA +fragments, method); the origin `Content-Type` fork (HTML vs `text/x-component` vs plain +URL replacer); the enabled-integration set; the build-time tsjs content hash. + +**Two pre-existing holes, worth filing regardless of this work:** the consent-denied / +bot / prefetch / no-slot variant keeps the origin's cacheability while still carrying +per-user `x-geo-*`; and the RSC-versus-HTML split is unguarded because TS never reads +`RSC` or `Next-Router-*`. + +**Invalidation signals TS has today:** config push changing slots — none; consent change +— request-side, belongs in the cache key; experiment rollover — origin-side only; +article edit — origin `Cache-Control`; tsjs rebuild — content hash, already in the URL; +integration enable/disable — none. + +--- + +## Appendix D — test inventory + +**Coverage gaps to close before Stage 2, not after.** No test exercises the hold +together with post-processors — `streaming_html_with_post_processors_rewrites_body` +(`publisher.rs:7966`) and `document_state_placeholders_substitute_through_accumulating_path` +(`publisher.rs:8043`) both pass `dispatched_auction: None`, which is exactly the Next.js +configuration in question. No parity coverage of the hold or bids injection either; +`parity.rs` has ten multi-adapter tests including `publisher_proxy_fallback_parity` +(`:762`), but none reaches the hold — extend that rather than building a second harness. + +`geo_header_parity_on_all_responses` (`parity.rs:613`) encodes the all-responses +invariant Stage 3b narrows, but currently covers only +`/.well-known/trusted-server.json`, `POST /auction`, and `POST /verify-signature`, and +asserts the boolean `x-geo-info-available` rather than per-user values — so it may need +no change. Check deliberately. + +**Twelve `publisher.rs` tests change with hold removal:** four die (`:5492`, `:5546`, +`:5630`, `:5649`); three assert hold-injected bids (`:6915`, `:6979`, `:7748`); two FCP +guards go vacuous (`:7306`, `:7341`); three are conditional on dispatch/collect +(`:5358`, `:7046`, `:7575`). Dead helpers: `ChunkedReader` (`:4226`), +`RecordingProcessor` (`:4252`). + +`html_processor.rs:1601` and `:1636` pre-populate the mutex directly — they stay green +while production injects empty bids, which is precisely why the fail-loud companion +exists. + +--- + +## Appendix E — ESI implementation notes + +For if and when D1's revival condition is met. + +Pin `esi = "0.7"`; pre-1.0, irregular cadence, two yanked betas in the 0.7 line. + +**Use `process_stream`, not the wrappers.** `process_response` and +`process_response_streaming` consume `self` _and_ send the response themselves, taking +ownership away from the finalize / `ec_finalize` / apply-effects ordering. + +**Order it esi → lol_html**, never the reverse, via a newtype implementing `io::Write` +that forwards to `HtmlRewriter::write`, with `end()` after `process_stream` returns. +Mind the `StreamingBody`-is-a-`BufWriter` hazard already recorded for this repo: esi +flushes after each parse batch, so any adapter in between must propagate `flush()`. + +**Always supply a custom fragment dispatcher.** The built-in one builds a dynamic +backend per URL host and panics on a hostless URL; dynamic backends are also the known +Viceroy local-dev failure mode here. Signature is +`Fn(Request, Option) -> Result` — `Fn`, not `FnMut`, so +captured counters need `Cell`/`RefCell`. Map the maxwait onto the quantized +backend-timeout scheme from #847. Fragment concurrency is free: includes dispatch at +parse time and harvest through one `select()` pool. + +**Streaming mode loses** `$add_header`, `$set_response_code`, `$set_redirect`, and the +auto `Cache-Control` from fragment TTLs — all announced via `println!`, not `log`. + +**Config explicitly:** `with_escaped(false)` for non-HTML templates; `with_chunk_size` +aligned to existing chunking, not the 16 KB default. + +**DCA off, and asserted off.** Defaults are `DcaMode::None` and +`inherit_parent_dca: false`, but set both explicitly — pre-1.0 defaults can move and +this one fails open. Rationale is the SSRF vector in [§2](#2-why--the-three-findings). +`max_include_depth` and `function_recursion_depth` bound the blast radius; they do not +close the hole. + +**Error semantics, non-obvious:** `alt` is attempted before `onerror="continue"` takes +effect; `` runs _all_ attempts in document order and concatenates every +non-failed output — not first-success-wins, so primary/fallback pairs render both; an +include with `onerror="continue"` inside `` never marks that attempt +failed, suppressing `except`. Wrap `ESIError` in `Report<...>` via `change_context()`. + +**Single include, not per-slot.** The auction is one operation producing all slots' +bids; there is no per-slot TTL or partial-failure boundary to exploit. + +**Before committing:** `cargo check-fastly` with `esi` added on Rust 1.95.0 / +`wasm32-wasip1`, and confirm the root lockfile does not desync from the +integration-tests lockfile on shared `regex`, `bytes`, `log`. + +--- + +## Appendix F — deferred open items + +Implementation-level, for unscheduled work only. The decisions that need a human are in +[§9](#9-decisions-needed-from-this-review). + +1. Should `collect_non_html_auction` (`publisher.rs:2388`) be removed with the hold or + kept? It is independently reachable and collects before any byte streams. +2. Is `body_close_hold_loop_stream` (`publisher.rs:2109`, no production caller) safe to + delete, or is the buffered-adapter streaming cutover (#495) still on the roadmap? +3. Does hidden-tab behaviour (rAF unserviced while hidden) interact badly with a bids + timeout that could burn freshness before the rAF fires? +4. Fastly's pending-request semantics when a `DispatchedAuction` drops mid-flight — + unverified; relevant only if dispatch/collect survives. +5. Does `stale-if-error` on a cached root serve acceptable content, given stale HTML + carries stale slot markup? Product call, surfaced by Stage 0. +6. The googletag shim discards listeners queued before it loads, breaking third-party + viewability tooling (#1009 Part 1). Not filed. Should be. + +--- + +## Appendix G — code-grounded seams + +All pinned to `cfb98f4`. + +| Concern | Location | +| ------------------------------------------- | ------------------------------------------------------------------------------- | +| Eligibility decision | `publisher.rs:2651`, `:2660` | +| `is_navigation_request` | `http_util.rs:73-98` | +| Auction dispatch (pre-origin, non-blocking) | `publisher.rs:2698-2760` | +| Auction overlap intent | `auction/orchestrator.rs:950-952` | +| Auction timeout (500 ms) | `settings.rs:5000`; `trusted-server.example.toml:174` | +| Conditional/range header strip | `publisher.rs:2832-2836` | +| Origin cache bypass | `publisher.rs:2866-2868` | +| Origin 304 → 502 guard | `publisher.rs:2894-2916` | +| `adSlots` build (per-URL) | `publisher.rs:2920`, `:3558-3577`, `:3501-3525` | +| Uncacheable stamp | `publisher.rs:2945-2963` | +| `` hold — sync / async / Fastly lazy | `publisher.rs:2235` / `:2109` / `:1318-1390` | +| Hold buffer | `publisher.rs:2177-2218` | +| Auction collect (HTML / non-HTML) | `publisher.rs:2431` (emits `:2456`) / `:2388` (emits `:2410`) | +| Abandonment emitter | `publisher.rs:2360` | +| Bids script build | `publisher.rs:3438-3491` | +| `/_ts/page-bids` | `publisher.rs:3611`, handler `:3723`, auction `:3903` | +| Injection seams (head / body-close) | `html_processor.rs:310-363` / `:381-395` | +| Post-processor buffering | `html_processor.rs:62-94` | +| Next.js post-processor registration | `integrations/nextjs/mod.rs:107` | +| Max buffered body (16 MB) | `settings.rs:77-79` | +| EC cookie issuance policy | `ec/finalize.rs:86-107` | +| Cookie-privacy net | `response_privacy.rs:20-61` | +| Geo response headers | `adapter-fastly/src/middleware.rs:194-200` | +| Cacheable-header precedent | `http_util.rs:294-311` | +| No purge permission | `adapter-fastly/src/management_api.rs:12` | +| Client initial-ad gate | `js/lib/src/integrations/gpt/index.ts:536-555` | +| `adInit` bid application | `js/lib/src/integrations/gpt/index.ts:566`, `:652`, `:657-661` | +| Client SPA auction hook | `js/lib/src/integrations/gpt/index.ts:806`, `:859`, `:892-949` | +| Prior design that introduced the killers | `docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md` | From 40de13e50a7ce7dc98de471989893df720019083 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 8 Aug 2026 15:36:36 +0530 Subject: [PATCH 166/395] Correct the hold-cost argument in the ESI cacheable-root design The strongest claim in the previous revision was wrong. It argued that on a Next.js publisher lol_html never sees the closing body tag until the final chunk, so the auction has the whole download plus rewrite to finish, and concluded that no timing data was needed. The hold does not key off lol_html at all. BodyCloseHoldBuffer::push scans the decoded origin input for the closing tag, and hold_collect_close_tail awaits collect_stream_auction the moment it appears, before post-processing runs. Post-processor buffering is irrelevant to when the hold fires, so the argument applied to every publisher or to none. What survives is the general form: the hold costs max(0, A - T) where T is origin TTFB plus transfer to the closing tag. That needs measurement rather than inference, so Step C now measures the hold directly via hold_wait_ms instead of comparing origin fetch against auction duration through a proxy model. The verdict table follows. Stage 0 becomes an operator flag rather than a code deletion. The risk it gates is cache poisoning, where rollback speed dominates diff size, and a config push reverts in seconds where a release does not. Also: the Vary precondition now covers client Cookie, origin Set-Cookie, and Authorization, which are a larger exposure than the RSC split it previously addressed; a Vary failure is recorded as a live production defect, since RSC fetches already transit the read-through cache; the auction timeout citation pointed at a test fixture rather than the real resolution order; and appendices B, C, E and F are condensed, since they specified work the document recommends against scheduling. --- ...08-esi-cacheable-root-validation-design.md | 425 ++++++++++-------- 1 file changed, 236 insertions(+), 189 deletions(-) diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md index 4e456e2d0..a2e23b4f9 100644 --- a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -33,7 +33,7 @@ about three days of measurement. | --- | -------------------------------------------------------------------------------------------------------------------------------- | ------------- | | D1 | **ESI is deferred.** Revival condition: #418 resolved _and_ the `window.load` gate removed. Not a rejection — a dated condition. | Eng + product | | D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | -| D3 | **Approve Stage 0** (remove `with_cache_bypass`), subject to the origin-`Vary` check in §3. | Eng | +| D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to the `Vary` check in §3. | Eng | | D4 | **Stages 1–2 queue behind the SSAT price defect and #418.** Stages 3b–5 unscheduled. | Product | Rationale for D4 in [§8](#8-priority). Everything this document recommends _against_ @@ -55,10 +55,10 @@ client fetch — no round trip — is worth nothing while bids are not consumed `window.load`. Separately, enabling ESI's Dynamic Content Assembly would be an SSRF vector: bid payloads carry partner-controlled creative markup, so an SSP could embed `` and make the edge fetch an arbitrary URL. Details in -[Appendix E](#appendix-e--esi-implementation-notes). +[Appendix E](#appendix-e--esi-notes-condensed). **The auction is already out of band; the hold is ~free.** It is dispatched _before_ -the origin fetch and does not block ([publisher.rs:2698-2701](../../../crates/trusted-server-core/src/publisher.rs#L2698-L2701)), +the origin fetch and does not block — dispatched at [publisher.rs:2751-2755](../../../crates/trusted-server-core/src/publisher.rs#L2751-L2755), sent at [:2870](../../../crates/trusted-server-core/src/publisher.rs#L2870) — with a 500 ms budget. The actual cost is `with_cache_bypass` ([publisher.rs:2867](../../../crates/trusted-server-core/src/publisher.rs#L2867)), which forces every ad-eligible navigation to miss the Fastly readthrough cache. @@ -93,13 +93,37 @@ path that already emits `public, s-maxage` twice and look for `x-cache`/`age` on TS's _own_ response. **Gates the Stage 3a/3b split** — see [§7](#7-deferred-work-specified-not-scheduled). -**Step C — server-side latency breakdown (1 day + a measurement window).** Emit four -timings per ad-eligible navigation: origin fetch duration (this is `O`, the quantity -the model lacks), auction collect duration, rewrite duration, total. Capture with the -bypass both on and off. - -- **Mechanism: `Server-Timing`.** Chosen, not offered — it needs no new plumbing and is - readable from the same browser harness that produced #1009's numbers. +**Step C — measure the hold directly (1 day + a measurement window).** + +The hold's cost is literally the duration of one `.await`: `collect_stream_auction` at +[publisher.rs:793](../../../crates/trusted-server-core/src/publisher.rs#L793), plus the +two EOF variants in `hold_finish_ready_segments` and `hold_finish_tail_segments`. Two +`Instant`s around it yield **`hold_wait_ms`** — the number this entire document is +arguing about, measured rather than modelled. + +Emit two timings per ad-eligible navigation: + +| Metric | Why | +| ----------------- | ---------------------------------------------------------------------- | +| `hold_wait_ms` | **The decision.** How long the response was actually held for bids. | +| `origin_fetch_ms` | Attribution — how much of the win Stage 0 can claim. Origin TTFB only. | + +`hold_wait_ms` replaces the proxy comparison an earlier draft proposed. Comparing `O` +against `A` was an indirect way of asking "does the hold block?"; this asks it directly, +costs less to build, and removes the modelling error corrected in +[§6.2](#62-what-the-hold-actually-costs). + +Deliberately not measured: auction collect duration is already instrumented +(`OrchestrationResult::total_time_ms`, `auction/orchestrator.rs:285`, flowing to +`auction_events_raw`) — read it, don't rebuild it. Rewrite duration decides nothing and +would mean touching two finalizers. + +- **Mechanism: a `log::info!` line behind a debug flag, not `Server-Timing`.** A response + header would in fact work for the origin-fetch figure — that value is known before + headers commit — but a server-side log needs no browser harness to collect it, `log` is + this project's instrumentation crate, and the auction path already measures itself with + `web_time::Instant`. Gate it behind config: one line per eligible navigation is real log + spend and the instrumentation is temporary. - **Sample: enough navigations per arm to separate the medians with confidence**, across both page types, and state the N alongside any result. #1009's sample was small enough that its conclusion did not survive contact with the code; replacing it with another @@ -107,22 +131,54 @@ bypass both on and off. **Step C has two outcomes, both actionable:** -| Outcome | Meaning | Effect on staging | -| -------------------------- | --------------------- | ------------------------------------------------------------ | -| `O` materially exceeds `A` | The model in §6 holds | Proceed as staged: Stage 0 primary, Stage 2 protects its win | -| `A` exceeds `O` | The hold _is_ costing | **Staging inverts** — Stage 2 primary, Stage 0 secondary | +| `hold_wait_ms` median | Meaning | Effect on staging | +| --------------------- | ----------------------- | ------------------------------------------------------------ | +| Near zero | The hold is free | Proceed as staged: Stage 0 primary, Stage 2 protects its win | +| Materially non-zero | The hold **is** costing | **Staging inverts** — Stage 2 primary, Stage 0 secondary | The work does not change; its order and justification do. **The staging in §7 is -conditional on this measurement.** +conditional on this measurement**, and the second outcome is a live possibility rather +than a formality — §6.2's argument for the first is weaker than an earlier draft claimed. -Step C also yields the client fetch latency that sets Stage 1's bids timeout, replacing -an invented constant. +Stage 1's bids-fetch timeout still needs a measured client-side figure rather than an +invented constant, but Step C is server-side and does not supply it. Capture it from the +browser harness when Stage 1 is actually scheduled. --- ## 4. Stage 0 — the only build item recommended now -Remove `with_cache_bypass` at [publisher.rs:2867](../../../crates/trusted-server-core/src/publisher.rs#L2867). +Stop bypassing the read-through cache on ad-eligible navigations +([publisher.rs:2867](../../../crates/trusted-server-core/src/publisher.rs#L2867)). + +**Ship it as an operator flag, not a deletion.** Add +`publisher.bypass_origin_cache`, defaulting to today's behaviour, in the same release as +the Step C instrumentation. Then turn it off with `ts config push`. + +The diff is slightly larger than deleting a line, and that is the point. The risk being +gated here is **cache poisoning** — serving one representation in response to a request +for another. For that class of failure, rollback speed dominates diff size: a config push +reverts in seconds, a release does not. The flag also buys an A/B on a byte-identical +build, removing build difference as a confound in the very measurement this depends on, +and allows flipping for a tester-cookie population before all traffic. + +Retire the flag once the change has held: flip the default, then delete the setting and +its branch. A temporary flag left in place becomes permanent configuration surface. + +### What to watch after the flip + +Two regression signals, both checked before the win is: + +- **`unexpected_origin_304` abandonment rate.** That reason + ([publisher.rs:2894-2916](../../../crates/trusted-server-core/src/publisher.rs#L2894-L2916), + emitted via `emit_abandoned_auction` at `:2360`) exists precisely because the ad-stack + path refuses cached and conditional origin responses. Re-enabling the cache is what + could revive it. **Any non-zero rate is a rollback signal** — it means a 304 is reaching + TS that the conditional-header strip was supposed to make impossible. +- **Representation mixing.** Spot-check that HTML navigations still return HTML and RSC + fetches still return `text/x-component`. A mismatch means the `Vary` risk materialized + despite a PASS verdict. Roll back immediately; this is cache poisoning, not a + performance regression. **Why it is safe in principle.** The conditional-header strip runs 34 lines earlier under the same gate ([publisher.rs:2832-2836](../../../crates/trusted-server-core/src/publisher.rs#L2832), @@ -146,11 +202,35 @@ The classification is also not airtight: `is_navigation_request` falls back to t weaker — `fetch()` can set Accept: text/html"_ ([http_util.rs:84-88](../../../crates/trusted-server-core/src/http_util.rs#L84)). +**A FAIL is not merely a Stage 0 blocker — it is a live production defect.** RSC fetches +already transit the read-through cache today, because they never set the bypass. If the +origin varies on `Next-Router-*` without declaring it, TS is cross-serving RSC variants +right now. On a FAIL, file that immediately and treat "ask the origin to declare `Vary`" +as urgent rather than as the cheaper of two options. + +**The `Vary` check is necessary but not sufficient.** Turning the read-through cache on +for HTML navigations exposes three things a representation check does not cover, and all +three are a larger class than the RSC split: + +- **Client `Cookie`.** TS forwards client cookies to origin unchanged — there is no + `COOKIE` strip on the publisher path. Any cookie-personalized HTML (logged-in state, + paywall meter, publisher-side A/B assignment) becomes cross-servable unless the origin + declares `Vary: Cookie` or marks those responses private. +- **Origin `Set-Cookie`.** If the origin emits `Set-Cookie` alongside a shared-cacheable + `Cache-Control`, the read-through cache can replay one visitor's cookie to the next. + TS's own privacy net downgrades **TS's** response — it runs after the cache has already + stored the origin's. +- **`Authorization`.** #1009 describes a basic-auth-gated deployment. Responses to + authorized requests entering a shared cache needs its own check. + +So Step A must capture `Cache-Control` and `Set-Cookie` too, and repeat each request with +and without a session cookie. Same minutes of work; closes the bigger hole. + **Two effort branches, and Step A decides which:** | Step A result | Stage 0 is… | Effort | | ---------------------- | --------------------------------------------- | ------ | -| Origin declares `Vary` | a one-line deletion plus test updates | 1–2 d | +| Origin declares `Vary` | the flag, its tests, then a config push | 1–2 d | | Origin does **not** | a TS-side cache-key discriminator — a feature | 4–8 d | The discriminator is the safer design either way, because it keys on the headers that @@ -208,33 +288,54 @@ changes here is its _causal weight_. Likewise, #1009's own observation that TS _ the auction cost from client-side to server-side rather than adding new work"_ is the argument for client-fill, which the issue then declines in favour of ESI. -### 6.2 Why the hold is free — the strong form first +### 6.2 What the hold actually costs + +**An earlier draft of this section claimed a stronger argument than the code supports. +It was wrong, and the correction matters.** -On a Next.js publisher, `lol_html` never sees the `` end tag until the **final** -chunk: with any post-processor registered, `HtmlWithPostProcessing` accumulates and -emits nothing before then ([html_processor.rs:62-65](../../../crates/trusted-server-core/src/html_processor.rs#L62)), -and the Next.js integration always registers one when enabled -([nextjs/mod.rs:107](../../../crates/trusted-server-core/src/integrations/nextjs/mod.rs#L107)). +The hold does not key off `lol_html` at all. `BodyCloseHoldBuffer::push` +([publisher.rs:2190-2202](../../../crates/trusted-server-core/src/publisher.rs#L2190)) +scans the **decoded origin input** for ` Dispatch precedes the origin fetch, so the hold costs `max(0, A − T)`, where `A` is the +> auction collect duration and `T` is origin TTFB plus body transfer up to the `` +> byte. Since `` sits at the end of a document, `T` is close to the full download. -The weaker, general form, for publishers with no post-processor registered: because -dispatch precedes the origin fetch, the hold costs `max(0, A − O)`, which is zero -whenever the origin build `O` exceeds the auction budget `A`. +`A` is bounded by `auction_timeout_ms`, resolved as +`creative_opportunities.auction_timeout_ms` falling back to `auction.timeout_ms` +([publisher.rs:2680-2684](../../../crates/trusted-server-core/src/publisher.rs#L2680-L2684)) +— check the resolution order against your own config rather than trusting a number; the +shipped example sets different values at each level. + +**This is a claim requiring measurement, not a proof.** §3 Step C measures the hold's +cost directly rather than inferring it. + +A finding that does survive, and belongs with [the ceiling](#64-the-ceiling): because +`HtmlWithPostProcessing` withholds all output until the final chunk, the streaming-prefix +design at [publisher.rs:1343-1348](../../../crates/trusted-server-core/src/publisher.rs#L1343-L1348) +— whose comment promises "the client receives the document up to `` while the +auction rides alongside transfer" — is **inert on a Next.js publisher**. Every +`step.ready` yields empty bytes. That comment is misleading on exactly the publisher +under discussion. ### 6.3 The quantity nobody has measured -Write the origin build time under `Pass` as `O`. Recovery depends on it, and it has -never been captured. #1009's timings cannot supply it: they compare a POP hit against a +Write the fetch time under `Pass` as `O`. Recovery depends on it, and it has never been +captured. #1009's timings cannot supply it: they compare a POP hit against a shield-served fetch, both of which are _cached_ paths, whereas `CacheOverride::Pass` -bypasses every layer and reaches the true origin. The two are different quantities. +bypasses TS's read-through cache and its shield. + +Note `Pass` bypasses **TS's** caches only. It has no authority over any CDN the publisher +runs in front of their own origin — and #1009's `x-cache: MISS, MISS` on the TS-on arm +hints one may exist. So `O` may not be origin build time at all. Since `O` is the single +quantity this model depends on, that ambiguity is worth resolving in Step C rather than +assuming. What follows from code alone, without any number: @@ -283,7 +384,7 @@ navigation generation 0. Endpoint, same-origin gate, wire shape, and client cons already exist. Three decisions must be made before planning: the `slots: []` precedence rule when head-open already injected a non-empty `ts.adSlots`; the new terminal-event emission point; and whether the dispatch/collect split survives at all. Plumbing detail -in [Appendix B](#appendix-b--stage-1-plumbing). Estimated 8–13 d, low-to-medium +in [Appendix B](#appendix-b--stage-1-plumbing-condensed). Estimated 8–13 d, low-to-medium confidence, uncertainty concentrated client-side. Three companions are mandatory, not optional: **suppress the server bids script @@ -320,7 +421,7 @@ body the document is no longer per-user. Record it as a deliberate decision. which shared-cached replays one visitor's geo to the next; geo is not a request header so suppression is the only option. Second, `Vary`: the publisher path emits none, and at least eleven request signals change the rewritten bytes for one URL — five are per-user -and can never be shared-cached ([Appendix C](#appendix-c--vary-signal-inventory)). +and can never be shared-cached ([Appendix C](#appendix-c--vary-signals-condensed)). **Also gated on Step B**: if nothing consumes TS's response headers, this tier is inert until a topology change. @@ -355,13 +456,18 @@ a path that 404s in a fresh checkout. ## 8. Priority -**Run §3 Steps A–C now, regardless of everything else.** Under three days combined, -useful independent of this effort, and Step C's instrumentation is a permanent -operational asset. +**Run §3 Steps A–C now, regardless of everything else.** Under three days combined, and +useful independent of this effort. Step C's instrumentation is deliberately temporary and +config-gated; if these timings become a standing regression gate, the right home is the +access-log telemetry already scaffolded but unwired in `TinybirdSettings` +(`settings.rs:1718-1731` — `access_enabled`, `access_dataset`, and a sample rate, with the +comment that it is _"rejected until an access-log emitter is wired"_). That is a +follow-on, not part of this work. -**Stage 0 next.** Small, gated on a `curl`, and it reverses an origin-load cost the -prior design explicitly accepted. Closer to a defect fix than an optimization — TS -opted out of a cache it did not need to opt out of. +**Stage 0 next**, shipped as the operator flag in §4 rather than a deletion. Gated on a +`curl`, reverses an origin-load cost the prior design explicitly accepted, and rolls back +with a config push. Closer to a defect fix than an optimization — TS opted out of a cache +it did not need to opt out of. **Stages 1–2 queue behind the correctness defects.** Their failure mode is silent revenue loss, against a publisher whose ads currently fill reliably. The SSAT price @@ -384,7 +490,7 @@ whatever hydration gate lands, and doing that twice is waste. human to make. Implementation-level open items for unscheduled work are in -[Appendix F](#appendix-f--deferred-open-items). +[Appendix F](#appendix-f--deferred-open-items-condensed). --- @@ -400,14 +506,14 @@ Rows 1–6 are in [§6.1](#61-corrections-to-1009s-premises). The remainder: **`should_run_ad_stack` carries four meanings across six sites:** -| Line | Meaning | Disposition | -| --------------------------------------------------------------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------ | -| [2651](../../../crates/trusted-server-core/src/publisher.rs#L2651), `:2660` | eligibility and auction gate | keep | -| [2832-2836](../../../crates/trusted-server-core/src/publisher.rs#L2832) | strip `If-None-Match`, `If-Modified-Since`, `Range`, `If-Range` | **keep** — needed for any injection | -| [2866](../../../crates/trusted-server-core/src/publisher.rs#L2866) | `with_cache_bypass()` | **remove** — [§4](#4-stage-0--the-only-build-item-recommended-now) | -| [2894](../../../crates/trusted-server-core/src/publisher.rs#L2894) | 304 → 502 guard | keep as safety net | -| [2920](../../../crates/trusted-server-core/src/publisher.rs#L2920) | build `adSlots` | keep — per-URL | -| [2945](../../../crates/trusted-server-core/src/publisher.rs#L2945) | strip cacheability | **replace** — Stage 3a | +| Line | Meaning | Disposition | +| --------------------------------------------------------------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| [2651](../../../crates/trusted-server-core/src/publisher.rs#L2651), `:2660` | eligibility and auction gate | keep | +| [2832-2836](../../../crates/trusted-server-core/src/publisher.rs#L2832) | strip `If-None-Match`, `If-Modified-Since`, `Range`, `If-Range` | **keep** — needed for any injection | +| [2866](../../../crates/trusted-server-core/src/publisher.rs#L2866) | `with_cache_bypass()` | **make operator-controlled** — [§4](#4-stage-0--the-only-build-item-recommended-now) | +| [2894](../../../crates/trusted-server-core/src/publisher.rs#L2894) | 304 → 502 guard | keep as safety net | +| [2920](../../../crates/trusted-server-core/src/publisher.rs#L2920) | build `adSlots` | keep — per-URL | +| [2945](../../../crates/trusted-server-core/src/publisher.rs#L2945) | strip cacheability | **replace** — Stage 3a | **Cache tiers.** T1 backend readthrough (already available; TS opts out). T2 TS-owned template cache (adds KV latency, eventual consistency, a full invalidation design). @@ -416,87 +522,58 @@ Compute is not invoked on a HIT). T2 and T3 both introduce a cache TS cannot pur --- -## Appendix B — Stage 1 plumbing - -All references below are `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` -unless the filename says otherwise. - -**Client plumbing.** `pageBidsEndpoint` (`index.ts:925`), `requestPageBids` -(`index.ts:927-944`), and the `inflight` / `currentPath` / `lastAppliedPath` state -(`index.ts:910`, `:915`, `:921`) are closure-trapped in `installSpaAuctionHook` and must -be hoisted to module scope so one abort domain covers both generations. Do **not** route -the initial load through `onNavigate`: `index.ts:947` no-ops it and `index.ts:949` -increments `navGeneration`, cancelling generation 0 at the guards on `index.ts:538` and -`:541`. - -**Suppress the server bids script.** Today the body end-tag handler is gated on -`has_slots`, which stays true because `adSlots` is still injected — so it would emit -`build_empty_bids_script()`, which calls `scheduleInitialAdInit({})` and assigns -`ts.bids = {}` synchronously at `index.ts:539`. Whether real bids survive would then -depend on unspecified ordering against the client fetch. The gate must become "did this -response carry bids," not "does this page have slots." - -**Gate restructuring — the real work.** `adInit` snapshots bids at call time -(`index.ts:566`) and applies `hb_*` targeting at `index.ts:657-661`; the -`slotRenderEnded` listener's live read at `index.ts:712` serves adm injection only and -cannot retarget a requested slot. **Bids arriving after `adInit` are lost.** -`installScheduleInitialAdInit` becomes a two-condition join — hydration-ready AND -bids-settled — with a bounded timeout that fires `adInit` untargeted rather than -stranding the slot. Derive the timeout from §3 Step C; `SPA_SLOT_WAIT_MS = 2000` is -precedent but was derived for DOM readiness, not network. This composes badly with the -958 branch's poll-and-grace gate — two timeout budgets in series. - -**Server contract.** `handle_page_bids` runs a fresh `run_auction` -(`publisher.rs:3903`) tagged `AuctionSource::SpaNavigation` (`publisher.rs:3859`). Not a -drop-in — it cannot reuse in-flight dispatched requests. Required: navigation-path -dispatch **suppressed** (running both doubles SSP/APS spend); a new `AuctionSource` for -initial loads **plus the mechanism that delivers it** — a request header alongside the -existing `X-TSJS-Page-Bids` marker, behind the same-origin gate, or it becomes a -caller-controlled telemetry-poisoning knob; and the `slots: []` precedence rule -(`publisher.rs:3975-3985`). - -**Telemetry.** Navigation `Completed` is emitted only from the two collect functions -(`publisher.rs:2410`, `:2456`); `Abandoned` only via `emit_abandoned_auction` -(`publisher.rs:2360`) across nine reasons, three of which live only inside the hold -helpers. The `[debug].auction_html_comment` `ts-debug` dump prepends onto the same -`ad_bids_state` string inside the collect function (`publisher.rs:2478`) and disappears -with the hold — relocate or retire deliberately. - -**Sequencing.** (a) decide bid delivery, (b) decide whether dispatch/collect survives, -(c) delete the hold. Doing (c) first produces the silent failure in [§5](#5-the-trap-in-the-deferred-work--read-this-before-scheduling-stages-12). -Leaving `OwnedProcessResponseParams` (`publisher.rs:1072-1086`) carrying five unread -auction fields is exactly that configuration. - -**Preserve the multi-member gzip guarantee.** `GzipDecodeReader` exists because -`flate2::read::GzDecoder` drops every member after the first, which can drop the -`` carrying the injection point. +## Appendix B — Stage 1 plumbing (condensed) + +Full detail lives in the plan when Stage 1 is scheduled. The decisions that must be made +before any of it is written: + +- **Suppress the server bids script entirely** under client-fill, not emit an empty one. + The body end-tag handler is gated on `has_slots`, which stays true; the gate must become + "did this response carry bids." +- **`adInit` snapshots bids at call time** (`gpt/index.ts:566`) and applies `hb_*` + targeting at `:657-661`. Bids arriving later are lost, so + `installScheduleInitialAdInit` must become a hydration-ready AND bids-settled join with + a bounded timeout that fires untargeted rather than stranding the slot. +- **Do not route the initial load through `onNavigate`** — `gpt/index.ts:949` increments + `navGeneration` and cancels generation 0. +- **`handle_page_bids` is not a drop-in.** It runs a fresh `run_auction` + (`publisher.rs:3903`) tagged `AuctionSource::SpaNavigation` (`:3859`) and cannot reuse + in-flight dispatched requests. Needs dispatch suppression (or spend doubles), a new + `AuctionSource` **plus the mechanism that delivers it**, and a `slots: []` precedence + rule (`:3975-3985`). +- **Telemetry moves with it.** Navigation `Completed` is emitted only from the collect + functions (`publisher.rs:2410`, `:2456`); `Abandoned` only via `emit_abandoned_auction` + (`:2360`). The `ts-debug` dump rides the same `ad_bids_state` string (`:2478`) and + disappears with the hold. +- **Sequencing is strict:** decide bid delivery, then whether dispatch/collect survives, + then delete the hold. Any other order produces the silent failure in [§5](#5-the-trap-in-the-deferred-work--read-this-before-scheduling-stages-12). --- -## Appendix C — `Vary` signal inventory +## Appendix C — `Vary` signals (condensed) -Needed for Stage 3b only. +Needed for Stage 3b only, which is unscheduled. -**Per-user — never shared-cacheable:** consent state (`euconsent-v2`, `__gpp`, +At least eleven request signals change the rewritten bytes for one URL. **Five are +per-user and can never be shared-cached:** consent state (`euconsent-v2`, `__gpp`, `__gpp_sid`, `us_privacy`, `Sec-GPC`, IP-derived jurisdiction); the GPT-diagnostics -`__Host-ts-console` cookie / `ts_console` query; the `tsjs.bids` payload (removed by -Stage 1, which is what makes the rest tractable); IP-derived geo; DataDome's request -filter, which can replace the document entirely. +`__Host-ts-console` cookie; the `tsjs.bids` payload (removed by Stage 1, which is what +makes the rest tractable); IP-derived geo; and DataDome's request filter, which can +replace the document entirely. -**Per-variant — safe in a cache key:** request host and scheme; `Accept-Encoding`; -request-class headers (`Sec-Fetch-Dest`, `Accept`, `Sec-Purpose`/`Purpose`, bot UA -fragments, method); the origin `Content-Type` fork (HTML vs `text/x-component` vs plain -URL replacer); the enabled-integration set; the build-time tsjs content hash. +**Six are per-variant and safe in a cache key:** request host and scheme, +`Accept-Encoding`, request-class headers, the origin `Content-Type` fork, the enabled +integration set, and the tsjs content hash. -**Two pre-existing holes, worth filing regardless of this work:** the consent-denied / -bot / prefetch / no-slot variant keeps the origin's cacheability while still carrying -per-user `x-geo-*`; and the RSC-versus-HTML split is unguarded because TS never reads -`RSC` or `Next-Router-*`. +Two pre-existing holes worth filing regardless of this work: the consent-denied / bot / +prefetch / no-slot variant keeps the origin's cacheability while carrying per-user +`x-geo-*`; and the RSC-versus-HTML split is unguarded because TS never reads `RSC` or +`Next-Router-*`. -**Invalidation signals TS has today:** config push changing slots — none; consent change -— request-side, belongs in the cache key; experiment rollover — origin-side only; -article edit — origin `Cache-Control`; tsjs rebuild — content hash, already in the URL; -integration enable/disable — none. +Invalidation signals TS has today: config push — none; consent change — request-side, +belongs in the cache key; experiment rollover — origin-side only; article edit — origin +`Cache-Control`; tsjs rebuild — content hash already in the URL; integration toggle — +none. --- @@ -528,73 +605,43 @@ exists. --- -## Appendix E — ESI implementation notes - -For if and when D1's revival condition is met. - -Pin `esi = "0.7"`; pre-1.0, irregular cadence, two yanked betas in the 0.7 line. - -**Use `process_stream`, not the wrappers.** `process_response` and -`process_response_streaming` consume `self` _and_ send the response themselves, taking -ownership away from the finalize / `ec_finalize` / apply-effects ordering. - -**Order it esi → lol_html**, never the reverse, via a newtype implementing `io::Write` -that forwards to `HtmlRewriter::write`, with `end()` after `process_stream` returns. -Mind the `StreamingBody`-is-a-`BufWriter` hazard already recorded for this repo: esi -flushes after each parse batch, so any adapter in between must propagate `flush()`. - -**Always supply a custom fragment dispatcher.** The built-in one builds a dynamic -backend per URL host and panics on a hostless URL; dynamic backends are also the known -Viceroy local-dev failure mode here. Signature is -`Fn(Request, Option) -> Result` — `Fn`, not `FnMut`, so -captured counters need `Cell`/`RefCell`. Map the maxwait onto the quantized -backend-timeout scheme from #847. Fragment concurrency is free: includes dispatch at -parse time and harvest through one `select()` pool. - -**Streaming mode loses** `$add_header`, `$set_response_code`, `$set_redirect`, and the -auto `Cache-Control` from fragment TTLs — all announced via `println!`, not `log`. - -**Config explicitly:** `with_escaped(false)` for non-HTML templates; `with_chunk_size` -aligned to existing chunking, not the 16 KB default. - -**DCA off, and asserted off.** Defaults are `DcaMode::None` and -`inherit_parent_dca: false`, but set both explicitly — pre-1.0 defaults can move and -this one fails open. Rationale is the SSRF vector in [§2](#2-why--the-three-findings). -`max_include_depth` and `function_recursion_depth` bound the blast radius; they do not -close the hole. - -**Error semantics, non-obvious:** `alt` is attempted before `onerror="continue"` takes -effect; `` runs _all_ attempts in document order and concatenates every -non-failed output — not first-success-wins, so primary/fallback pairs render both; an -include with `onerror="continue"` inside `` never marks that attempt -failed, suppressing `except`. Wrap `ESIError` in `Report<...>` via `change_context()`. - -**Single include, not per-slot.** The auction is one operation producing all slots' -bids; there is no per-slot TTL or partial-failure boundary to exploit. - -**Before committing:** `cargo check-fastly` with `esi` added on Rust 1.95.0 / -`wasm32-wasip1`, and confirm the root lockfile does not desync from the -integration-tests lockfile on shared `regex`, `bytes`, `log`. +## Appendix E — ESI notes (condensed) + +For if and when [D1](#1-decision-requested)'s revival condition is met. Expand then; +recording only what would otherwise be re-derived: + +- Pin `esi = "0.7"`. Pre-1.0, irregular cadence, two yanked betas in the 0.7 line. +- **Use `process_stream`, not the wrappers.** `process_response` and + `process_response_streaming` consume `self` _and_ send the response themselves, taking + ownership away from the finalize / `ec_finalize` ordering. +- **Order esi → lol_html**, never the reverse, via a newtype implementing `io::Write`. + Mind the `StreamingBody`-is-a-`BufWriter` hazard: esi flushes per parse batch, so any + adapter in between must propagate `flush()`. +- **Always supply a custom fragment dispatcher.** The built-in one builds a dynamic + backend per URL host and panics on a hostless URL; dynamic backends are also the known + Viceroy local-dev failure mode here. +- **DCA off, asserted explicitly** — not merely left at its default. Rationale is the SSRF + vector in [§2](#2-why--the-three-findings): partner-controlled creative markup would + become ESI-executable at the edge. +- **`` runs _all_ attempts and concatenates every non-failed output** — not + first-success-wins, so primary/fallback pairs render both. Least obvious behaviour in + the crate. +- Single include, not per-slot: the auction is one operation producing all slots' bids. --- -## Appendix F — deferred open items +## Appendix F — deferred open items (condensed) -Implementation-level, for unscheduled work only. The decisions that need a human are in +Implementation-level, for unscheduled work only. Decisions needing a human are in [§9](#9-decisions-needed-from-this-review). -1. Should `collect_non_html_auction` (`publisher.rs:2388`) be removed with the hold or - kept? It is independently reachable and collects before any byte streams. -2. Is `body_close_hold_loop_stream` (`publisher.rs:2109`, no production caller) safe to - delete, or is the buffered-adapter streaming cutover (#495) still on the roadmap? -3. Does hidden-tab behaviour (rAF unserviced while hidden) interact badly with a bids - timeout that could burn freshness before the rAF fires? -4. Fastly's pending-request semantics when a `DispatchedAuction` drops mid-flight — - unverified; relevant only if dispatch/collect survives. -5. Does `stale-if-error` on a cached root serve acceptable content, given stale HTML - carries stale slot markup? Product call, surfaced by Stage 0. -6. The googletag shim discards listeners queued before it loads, breaking third-party - viewability tooling (#1009 Part 1). Not filed. Should be. +Should `collect_non_html_auction` (`publisher.rs:2388`) go with the hold or stay? Is +`body_close_hold_loop_stream` (`:2109`, no production caller) safe to delete, or is the +buffered-adapter streaming cutover (#495) still live? Does hidden-tab rAF behaviour +interact badly with a bids timeout? What are Fastly's pending-request semantics when a +`DispatchedAuction` drops mid-flight? Does `stale-if-error` on a cached root serve +acceptable content given stale slot markup? And the googletag shim discards listeners +queued before it loads (#1009 Part 1) — not filed, should be. --- @@ -608,7 +655,7 @@ All pinned to `cfb98f4`. | `is_navigation_request` | `http_util.rs:73-98` | | Auction dispatch (pre-origin, non-blocking) | `publisher.rs:2698-2760` | | Auction overlap intent | `auction/orchestrator.rs:950-952` | -| Auction timeout (500 ms) | `settings.rs:5000`; `trusted-server.example.toml:174` | +| Auction timeout resolution | `publisher.rs:2680-2684` (creative_opportunities, else auction.timeout_ms) | | Conditional/range header strip | `publisher.rs:2832-2836` | | Origin cache bypass | `publisher.rs:2866-2868` | | Origin 304 → 502 guard | `publisher.rs:2894-2916` | From f15eaba7523aa640b78f2d0f533dc1e1543d4d10 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 8 Aug 2026 15:37:29 +0530 Subject: [PATCH 167/395] Add the measurement and Stage 0 implementation plan for #1009 Covers the spec's Steps A/B/C plus Stage 0. Stages 1-5 are out of scope and named as such, since the spec queues them behind the open correctness defects. Three investigations and one code task. Step A curls the origin for its Vary declaration and for cookie personalization, and gates everything downstream. Step B settles whether anything caches the service's own response by inspecting the Fastly topology rather than probing for an age header, and asks whether the publisher backend is shielded, which sizes the win and nothing else in the plan establishes. Step C instruments hold_wait_ms and origin_fetch_ms. The instrumentation goes in collect_stream_auction rather than at its three call sites. All three reach it, and it already destructures settings out of AuctionCollectDeps, so one edit covers every adapter with no new plumbing. The plan names hold_finish_ready_segments and hold_finish_tail_segments explicitly as sites not to instrument: neither awaits the collect, and doing so would double-count. Stage 0 ships as publisher.bypass_origin_cache defaulting to today's behaviour, then flips by config push. Adding that field breaks nine sites the diff does not suggest, including a live doctest, so they are enumerated. The win is measured client-side through the existing tester-cookie harness; origin_fetch_ms is TTFB only and is attribution, not outcome. Records two gotchas hit while writing it: prettier is not idempotent on markdown containing fenced markdown blocks, and it rewrites bare snake_case identifiers inside them as emphasis. Both fail CI gate 7. --- ...2026-08-08-1009-measurement-and-stage-0.md | 954 ++++++++++++++++++ 1 file changed, 954 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md new file mode 100644 index 000000000..c5f85aa4c --- /dev/null +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md @@ -0,0 +1,954 @@ +# #1009 Measurement and Stage 0 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Obtain the checks that gate the #1009 work, then turn off the redundant origin +cache bypass that the spec identifies as the actual TTFB cost — behind an operator flag, +so it rolls back with a config push rather than a release. + +**Architecture:** Two investigation tasks that produce recorded findings and no code; one +code task that adds a config-gated timing log and makes the cache bypass operator- +controlled; and one config change that flips it, gated on the first investigation. +Nothing here touches the auction, the `` hold, or bid delivery — those are +Stages 1–2 in the spec and are explicitly out of scope. + +**Tech Stack:** Rust 2024 edition, `wasm32-wasip1`, Fastly Compute, `web_time::Instant` +for wasm-safe timing, `log` for instrumentation, Viceroy for adapter tests. + +**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +(§3 Steps A/B/C and §4 Stage 0). Read §4 and §5 before starting Task 5. + +**Two prettier gotchas, both hit while writing this plan.** CI gate 7 +(`cd docs && npm run format`) runs `prettier --check` over all of `docs/`, so both bite. + +1. **Not idempotent on embedded markdown fences.** The first `--write` reformats the + outer document and the embedded ` ```markdown ` block only settles on a second pass. + If `--check` still warns immediately after a `--write`, run `--write` again before + concluding anything is wrong. +2. **It mangles bare `snake_case` identifiers inside fences**, reading the underscores as + emphasis and rewriting `origin_fetch_ms` to `origin*fetch_ms`. **Always wrap + identifiers in backticks**, including inside fenced blocks and table cells. + +--- + +## Background an implementer needs + +Trusted Server proxies a publisher's origin, rewrites the HTML at the edge to inject ad +slot definitions and a JS bundle, and runs a server-side ad auction. For requests that +are eligible for that ad stack, `publisher.rs` currently does three things to the origin +request and response that together make the page uncacheable: + +1. strips conditional and range headers so the origin must return a full body, +2. sets a **cache bypass** so the Fastly read-through cache is skipped entirely, and +3. strips every cacheability header from the response. + +The spec establishes that (2) is redundant given (1) — by the time the request reaches +the cache it is already unconditional, so a cache HIT returns a full body anyway — and +that (2) is the dominant cost. This plan makes (2) operator-controlled and then turns it +off, after first confirming that is safe. + +**Why it might not be safe:** RSC (React Server Component) requests and ordinary HTML +navigations share the same URL and are distinguished only by request headers. RSC +requests are not classified as navigations, so they already flow through the cache while +HTML navigations bypass it. Removing the bypass puts both under one cache key. If the +origin does not declare `Vary` for those headers, the cache could serve one +representation in response to a request for the other. Task 1 checks this. + +**Terms:** _POP_ = Fastly edge point of presence. _shield_ = a designated POP that +backs other POPs. _read-through cache_ = Fastly's cache on the backend request path. +_bypass / `Pass`_ = skip that cache. + +--- + +## File structure + +| File | Responsibility in this plan | +| ---------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` | **Create.** Recorded output of Tasks 1–2. Gates Task 5. | +| `crates/trusted-server-core/src/publisher.rs` | **Modify.** Timing log and the bypass flag (Task 3); tests (Task 5). | +| `crates/trusted-server-core/src/settings.rs` | **Modify.** `publisher.bypass_origin_cache` and `debug.publisher_timing` (Task 3). | +| `trusted-server.example.toml` | **Modify.** Document the new key (Task 5). | + +No new modules. No adapter changes: the `bypass_cache` platform capability and its +per-adapter mappings stay in place and keep their tests — the publisher-path call site +becomes operator-controlled rather than unconditional. + +## Task order and dependencies + +Only one edge is real. Do not serialize the rest. + +``` +Task 1 (origin Vary check) ──────┬──> Task 2 (appends to the findings file Task 1 creates) + │ + ├──> Task 5 (flip the flag) +Task 3 (instrumentation + flag) ─┘ +``` + +**Task 1 is externally blocked.** It needs the publisher origin hostname, which lives in +the operator's gitignored `trusted-server.toml`. Arrange access before starting, or the +plan stalls on its first step. + +Task 3 is independent and can start immediately. Task 2 only needs Task 1 far enough to +have created the findings document. Task 5 needs Task 1's verdict **and** Task 3's config +flag to exist. + +--- + +## Task 1: Step A — origin `Vary` check + +**Files:** + +- Create: `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` + +This task is an investigation. It writes no code and gates Task 5. + +- [ ] **Step 1: Get the origin URL** + +The publisher origin is operator config, not in the repo. Read it from the deployed +service config or ask the operator. Do **not** hardcode it into any committed file — the +findings document records the _result_, not the hostname. + +```bash +# The key is `publisher.origin_url` in the operator's trusted-server.toml +# (gitignored). Confirm the value before proceeding. +``` + +- [ ] **Step 2: Request the HTML representation and capture `Vary`** + +```bash +curl -sS -D - -o /dev/null "https:///" \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Accept: text/html' +``` + +Expected: response headers. Record whether a `Vary` header is present and its value. + +- [ ] **Step 3: Request the RSC representation at the same URL** + +```bash +curl -sS -D - -o /dev/null "https:///" \ + -H 'RSC: 1' \ + -H 'Accept: text/x-component' +``` + +Expected: a different `Content-Type` (`text/x-component`) than Step 2, proving the two +representations share a URL. Record `Vary` again. + +- [ ] **Step 4: Probe the `Next-Router-*` headers** + +Do not skip this. The PASS criterion below names these headers, and an implementer who +tests only HTML and `RSC` can record a PASS that is wrong — which routes to Task 5a, the +one outcome this plan calls dangerous. + +```bash +for H in 'Next-Router-Prefetch: 1' 'Next-Router-State-Tree: %5B%22%22%5D'; do + echo "--- $H" + curl -sS -D - -o /dev/null "https:///" \ + -H 'RSC: 1' -H "$H" \ + | grep -iE '^(vary|content-type|content-length|cache-control|set-cookie):' +done +``` + +Compare `Content-Type` and `Content-Length` against the plain `RSC: 1` request from +Step 3. If either differs, the origin varies on that header and `Vary` must name it. + +Capture `Cache-Control` and `Set-Cookie` on every request in this task, not just this +one — see Step 5. + +- [ ] **Step 5: Probe cookie personalization — the bigger hole** + +The representation check above covers RSC-vs-HTML. It does **not** cover the larger +class: TS forwards client cookies to origin unchanged, so any cookie-personalized HTML +(logged-in state, paywall meter, publisher-side A/B assignment) becomes cross-servable +once the cache is on. + +```bash +# Same URL, with and without a session cookie. Compare Content-Length and body hash. +for C in '' 'Cookie: '; do + echo "--- ${C:-no-cookie}" + curl -sS -D /dev/stderr -o - "https:///" \ + -H 'Sec-Fetch-Dest: document' -H 'Accept: text/html' ${C:+-H "$C"} \ + 2> >(grep -iE '^(vary|cache-control|set-cookie|content-length):' >&2) \ + | shasum +done +``` + +Three failure shapes, any of which blocks Stage 0 independently of the `Vary` verdict: + +- Bodies differ by cookie **and** `Vary` does not name `Cookie` → cross-serving of + personalized HTML. +- Origin emits `Set-Cookie` alongside a shared-cacheable `Cache-Control` → the cache can + replay one visitor's cookie to the next. TS's privacy net does not help; it downgrades + **TS's** response, after the cache has already stored the origin's. +- The deployment is `Authorization`-gated (as #1009 describes) and authorized responses + are cacheable → same problem, different header. + +- [ ] **Step 6: Request with the experiment header, if the operator uses one** + +Repeat Step 2 with the publisher's experiment header set to two different values. +Record whether the bodies differ and whether `Vary` names that header. + +- [ ] **Step 7: Record the finding** + +Create `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md`: + +```markdown +# #1009 measurement findings + +## Step A — origin `Vary` declaration + +**Date:** · **Checked by:** + +| Representation | `Content-Type` returned | `Content-Length` | `Vary` present? | `Vary` value | +| --------------------- | ----------------------- | ---------------- | --------------- | ------------ | +| HTML navigation | | | | | +| RSC | | | | | +| RSC + `Next-Router-*` | | | | | +| Experiment variant | | | | | + +**Cookie / auth exposure:** bodies differ by cookie? `Vary: Cookie` present? origin +`Set-Cookie` on a shared-cacheable response? `Authorization`-gated responses cacheable? + +**Verdict:** PASS / FAIL + +PASS = `Vary` names every request header the origin varies on (`RSC`, any `Next-Router-*` +or experiment header whose value changed the body, **and `Cookie` if bodies differ by +cookie**), and no `Set-Cookie` rides a shared-cacheable response. +FAIL = any of the above is unmet. + +**Consequence:** PASS → Task 5a (flip the flag). FAIL → Task 5b (cache-key +discriminator). See spec §4. + +**A FAIL is also a live production defect, not only a Stage 0 blocker.** RSC fetches are +not navigations, so they never set the bypass and **already transit the read-through +cache today**. If the origin varies undeclared on `Next-Router-*`, TS is cross-serving RSC +variants in production right now. File it immediately rather than deferring with Task 5b. +``` + +- [ ] **Step 8: Commit** + +CI gate 7 runs `prettier --check` across all of `docs/`, so format the findings file +before staging it — a filled-in markdown table will not be prettier-clean by hand. + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record origin Vary findings for #1009 Stage 0 gate" +``` + +--- + +## Task 2: Step B — what consumes TS's own response headers + +**Files:** + +- Modify: `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` — **created by + Task 1 Step 6.** If Task 1 has not reached that step, create the file with just its + `# #1009 measurement findings` heading rather than blocking. + +Investigation. Determines whether the spec's Stage 3b has a consumer. Does not gate +Task 5, but it appends to Task 1's findings document — do not run the two concurrently +against that file. + +- [ ] **Step 1: Pick a path that already emits shared-cache headers** + +`serve_static_with_etag` emits `public, max-age=300, s-maxage=300` plus +`Surrogate-Control` — see `crates/trusted-server-core/src/http_util.rs:294-311`. It backs +the `/static/tsjs=` bundle route (`publisher.rs:303`, `:322`). Use that URL against +the deployed service. + +- [ ] **Step 2: Request it twice and inspect for cache markers** + +```bash +URL="https:///static/tsjs=" +curl -sS -D - -o /dev/null "$URL" | grep -iE 'x-cache|age:|x-served-by|hit-state' +sleep 2 +curl -sS -D - -o /dev/null "$URL" | grep -iE 'x-cache|age:|x-served-by|hit-state' +``` + +Expected on the second request: if a cache sits in front of the Compute service, an `age` +greater than zero or an `x-cache` containing `HIT`. + +**The probe above is weak evidence** — absence of `age` is equally consistent with "no +cache" and "cold cache". **The topology check below is the actual answer; run it first and +skip the probe if it is conclusive.** + +```bash +fastly service list +fastly service-version list --service-id +# Look for a Delivery service fronting the Compute service, and for shielding +# configured on the service rather than only on the origin backend. +``` + +A Compute service with no Delivery service in front and no fronting shield does not have +its own output cached — that is the configuration the spec assumes, and this step exists +to confirm or refute it rather than to leave it assumed. + +**While you have the service open, answer a second question that matters more than this +task does:** is the _publisher backend_ shielded on the TS service? + +```bash +fastly backend list --service-id --version active +# Look for a shield on the publisher origin backend. +``` + +#1009's entire off-TS advantage came from a **shield** HIT, not a POP HIT. Whether +Stage 0 recovers a shield HIT or only a single-POP HIT changes the size of the win +materially, and nothing else in this plan establishes it. + +- [ ] **Step 3: Record the finding** + +Append to the findings document: + +```markdown +## Step B — consumers of TS's own response headers + +**Verdict:** SHARED CACHE PRESENT / NO SHARED CACHE + +**Evidence:** + +**Consequence:** NO SHARED CACHE → spec Stage 3b is inert until a topology change; +deprioritize it and ship only Stage 3a (browser caching). SHARED CACHE PRESENT → +Stage 3b gains a consumer AND the per-user `x-geo-*` header leak in spec §7 becomes an +active privacy exposure rather than a theoretical one. Escalate immediately in that case. +``` + +- [ ] **Step 4: Commit** + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record response-header cache consumer findings for #1009" +``` + +--- + +## Task 3: Step C — origin fetch timing, and the bypass flag + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/settings.rs` — `publisher.bypass_origin_cache`, + `default_bypass_origin_cache`, `debug.publisher_timing`, the `Publisher` `Default` impl, + eight test literals, and the `origin_host` doctest +- Modify: `crates/trusted-server-core/src/test_support.rs` (log-capture helper) +- Test: inside `mod ssat_cache_policy_tests` at `crates/trusted-server-core/src/publisher.rs:4541` + +**Use `web_time::Instant`, not `std::time::Instant`** — the workspace targets +`wasm32-wasip1` and `web_time` is the wasm-safe clock already used at +`crates/trusted-server-core/src/auction/orchestrator.rs:7`. + +### Two timings, and why these two + +Measure **`hold_wait_ms`** and **`origin_fetch_ms`**. Not the rewrite. + +`hold_wait_ms` is the decision. The hold's cost is literally the duration of one +`.await` — `collect_stream_auction` at `publisher.rs:793`, plus the two EOF variants in +`hold_finish_ready_segments` (`:869`) and `hold_finish_tail_segments` (`:896`). Two +`Instant`s around those calls answer "does the hold block?" directly, instead of +inferring it by comparing origin fetch against auction duration. + +`origin_fetch_ms` is attribution — how much of any win Stage 0 can claim. + +`rewrite_ms` decides nothing. Step C's verdict compares origin fetch against auction +collect, and the ceiling argument in spec §6.4 is structural — it needs no number. +Measuring the rewrite would mean instrumenting two finalizers +(`buffer_publisher_response_async` at `publisher.rs:1114`, and the +`async_stream::try_stream!` block at `publisher.rs:1286`), working around moves out of +`params` inside that block, and finding a correlation key that does not exist — +`OwnedProcessResponseParams` (`publisher.rs:1065-1087`) has no `request_path`, and adding +one means touching all 26 construction sites. + +None of that buys a decision. Skip it. If a rewrite figure is later wanted to set a +target, add it as a separate follow-on once the verdict is known. + +**Why a log line and not `Server-Timing`:** for `origin_fetch_ms` alone a response header +would in fact work — the value is known before headers commit. A log line is still +preferred because it is server-side (no dependence on a browser harness to collect it), +`log` is this project's instrumentation crate per `CLAUDE.md`, and the auction path +already measures itself the same way. The spec previously claimed `Server-Timing` cannot +work at all; that overbroad claim has already been corrected there. + +### Log volume — gate it + +The line sits after the origin send, so it fires for every publisher request that reaches +origin — tagged `ad_stack=false` for ineligible ones, not only for eligible navigations. +That is more useful for comparison and more log spend, and the instrumentation is +temporary either way. Gate it behind the existing debug surface rather than +emitting unconditionally: add a `#[serde(default)] pub publisher_timing: bool` to +`DebugConfig` (`crates/trusted-server-core/src/settings.rs:1872`), following +`ja4_endpoint_enabled` and `auction_html_comment` alongside it. Default `false`; enable +via `ts config push` for the measurement window, then disable. + +This also means the Step 1 test must set that flag in its settings fixture. + +The split is also what makes the Step 1 test achievable — `run_with_slots` +(`publisher.rs:4769`) invokes only `handle_publisher_request` and never drives either +finalizer, so a test asserting on a combined line could never pass. + +**What `origin_fetch_ms` actually measures.** `publisher.rs:2863-2865` sets +`.with_stream_response()` when the adapter supports it, so on Fastly `send()` returns at +response _headers_, not after the body downloads. `origin_fetch_ms` is therefore **origin +TTFB**, not full download time. Name it that way in the findings document. It is still +the correct before/after signal for Stage 0 — the bypass affects whether the request hits +a cache at all — but when comparing against auction `total_time_ms` in Step 9, compare +like with like and say which quantity each column holds. + +- [ ] **Step 1: Write the failing test** + +**Placement matters.** Add the test **inside `mod ssat_cache_policy_tests`** +(`publisher.rs:4541`), not the outer `mod tests` (`:4035`). Every helper it uses is +private to that nested module: `settings_with_enabled_auction_and_creative_opportunities` +(`:4684`), `article_slot` (`:4721`), `conditional_navigation_request` (`:4740`), +`queue_cacheable_html_response` (`:4752`), `run_with_slots` (`:4769`). Placed in the outer +module it will not resolve — and because two _other_ `article_slot` functions exist +(`:9593`, `:10276`) returning a different type, the failure surfaces as a confusing type +error rather than a missing-name error. + +**First, add the log-capture helper.** `crates/trusted-server-core/src/test_support.rs` +has none. Note its shape: the whole file is `#[cfg(test)] pub mod tests { … }`, so the +path is `crate::test_support::tests::capture_logs`, not `crate::test_support::capture_logs` +— see existing consumers at `auth.rs:103` and `config_payload.rs:48`. + +Two constraints the helper must respect or the test fails for unrelated reasons: + +- `log::set_boxed_logger` succeeds **once per process**. Install via a `OnceLock`/`Once` + and have `capture_logs()` return a guard that clears and then reads a shared buffer. +- Call `log::set_max_level(log::LevelFilter::Info)` or higher, or `log::info!` is filtered + out before it reaches the logger. +- **Do not have the guard hold the buffer's own `Mutex`.** The test body runs code that + calls `log::info!` on the same thread, and the logger must lock that same mutex to + append — `std::sync::Mutex` is not reentrant, so this **hangs** rather than failing. + Use two locks: a separate process-wide serialization mutex held by the guard, and the + buffer's own mutex taken and released per line by the logger. +- The buffer is process-global and every other concurrently-running `trusted-server-core` + test logs into it, so a `got: {captured}` diagnostic will be large. Assert with + `contains`, not equality. +- `log::set_max_level` is global for the test binary. Setting it to `Info` is fine, but it + affects every test in the process. + +```rust +#[tokio::test] +async fn eligible_navigation_logs_origin_fetch_duration() { + // Arrange + let logs = crate::test_support::tests::capture_logs(); + let mut settings = settings_with_enabled_auction_and_creative_opportunities(); + // The log line is gated; without this the assertions below can never pass. + settings.debug.publisher_timing = true; + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let _ = run_with_slots(&settings, &services, &slots, conditional_navigation_request()).await; + + // Assert + let captured = logs.contents(); + assert!( + captured.contains("publisher_timing"), + "eligible navigation should emit a publisher_timing log line, got: {captured}" + ); + assert!( + captured.contains("origin_fetch_ms="), + "publisher_timing should record origin_fetch_ms, got: {captured}" + ); +} +``` + +This test deliberately asserts only on the `publisher_timing` line. `run_with_slots` never +drives a finalizer, so `publisher_rewrite` is out of its reach — cover that separately if +at all, rather than contorting this test. + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation_logs_origin_fetch_duration -- --nocapture +``` + +Expected: FAIL — no `publisher_timing` in the captured logs. (Substitute your host +triple; core tests run natively for fast iteration. The Viceroy run comes in Step 6.) + +- [ ] **Step 3: Time the origin fetch** + +In `publisher.rs`, at the top with the other imports, add: + +```rust +use web_time::Instant; +``` + +Then wrap the origin send. The current code is at `publisher.rs:2870`: + +```rust +let mut response = match services.http_client().send(platform_request).await { +``` + +Change it to: + +```rust +let origin_fetch_start = Instant::now(); +let mut response = match services.http_client().send(platform_request).await { +``` + +and immediately after the `match` completes (after the existing `};` that closes it, +before the existing `log::debug!("Publisher origin response received: ...")` at `:2888`): + +```rust +let origin_fetch_ms = u64::try_from(origin_fetch_start.elapsed().as_millis()).unwrap_or(u64::MAX); +``` + +**Make the bypass config-driven in the same change.** This is what lets Stage 0 ship as a +config flip rather than a second deploy — see Task 5. Replace the block at +`publisher.rs:2866-2868`: + +```rust +// Single source of truth for the request and the log line below. Operator- +// controlled so the read-through cache can be re-enabled without a release; +// see docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md §4. +let cache_bypass = should_run_ad_stack && settings.publisher.bypass_origin_cache; +if cache_bypass { + platform_request = platform_request.with_cache_bypass(); +} +``` + +Add the setting to `Publisher` in `crates/trusted-server-core/src/settings.rs:29`, +**defaulting to today's behaviour** so this change is a no-op until deliberately flipped: + +```rust +/// Bypass the platform read-through cache on ad-eligible publisher navigations. +/// +/// `true` preserves the historical behaviour introduced by the SSAT 304-prevention +/// design. `false` lets those navigations use the read-through cache; the +/// conditional-header strip already guarantees a complete body on a cache HIT. +/// Temporary operator control for the Stage 0 rollout — remove once settled. +#[serde(default = "default_bypass_origin_cache")] +pub bypass_origin_cache: bool, +``` + +```rust +fn default_bypass_origin_cache() -> bool { + true +} +``` + +**Adding this field breaks nine sites. Update them in the same commit or Step 2 fails to +compile before it can produce the intended RED failure:** + +- The hand-written `Default` impl at `settings.rs:81-97`. +- Eight exhaustive test literals. The line numbers below anchor each + `let publisher = Publisher {` **opening**, not a field — add the new field inside each + brace: `settings.rs:3553`, `:3564`, `:3575`, `:3586`, `:3597`, `:3608`, `:3621`, + `:3635`. `clippy-fastly` runs `--all-targets`, so these gate lint too. +- The rustdoc example for `origin_host`, whose literal opens at `settings.rs:130`. + **This is a live doctest** and the host-triple test command below does not skip + doctests. + +While there, mirror the existing default-agreement test +`publisher_default_max_buffered_body_bytes_matches_config_default` (`settings.rs:3648`) — +it exists to catch a hand-written `Default` diverging from a serde default, which is +exactly the shape this field re-introduces. One assertion. + +Then emit the line, immediately after computing `origin_fetch_ms`, gated on the debug +flag from the section above: + +```rust +if settings.debug.publisher_timing { + log::info!( + "publisher_timing origin_fetch_ms={origin_fetch_ms} \ + cache_bypass={cache_bypass} ad_stack={should_run_ad_stack}" + ); +} +``` + +- [ ] **Step 4: Instrument `hold_wait_ms` — the decision metric** + +This is the number the whole effort turns on, and it needs **one edit in one function**. + +`collect_stream_auction` (`publisher.rs:2431`) is the only function that awaits the +auction collect, and all three call sites reach it: + +| Call site | Path | +| ------------------- | ------------------------------------------------------------ | +| `publisher.rs:793` | `hold_collect_close_tail` — Fastly lazy stream | +| `publisher.rs:2257` | `body_close_hold_loop`, EOF arm — Axum, Cloudflare, Spin | +| `publisher.rs:2311` | `body_close_hold_loop`, mid-stream arm — same three adapters | + +Instrument the callee, not the callers. It already destructures `settings` out of +`AuctionCollectDeps` (`:2436`), so the debug flag is in scope with no new plumbing, and +one edit covers every adapter. + +Wrap the `collect_dispatched_auction` await at `:2447-2449`: + +```rust + let hold_wait_start = Instant::now(); + let result = orchestrator + .collect_dispatched_auction(dispatched, services, &collect_ctx) + .await; + if settings.debug.publisher_timing { + let hold_wait_ms = + u64::try_from(hold_wait_start.elapsed().as_millis()).unwrap_or(u64::MAX); + log::info!("publisher_hold hold_wait_ms={hold_wait_ms}"); + } +``` + +`settings` here is `&&Settings` from the destructure — deref as needed; the compiler will +say so. + +**Do not instrument `hold_finish_ready_segments` (`:869`) or `hold_finish_tail_segments` +(`:896`).** Neither awaits the collect. The first returns `close_found` for its caller to +act on; the second delegates to `hold_collect_close_tail` at `:909`. Instrumenting them +would double-count. + +**Do not instrument the auction itself.** `OrchestrationResult::total_time_ms` +(`orchestrator.rs:285`, struct at `:1449`, per-provider at `:365`) already flows to +`auction_events_raw`. `hold_wait_ms` measures something different and more useful: how +long the _response_ waited, which is near zero when the auction finished during transfer +even though `total_time_ms` is large. + +- [ ] **Step 5: Run the test to verify it passes** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation_logs_origin_fetch_duration -- --nocapture +``` + +Expected: PASS. + +- [ ] **Step 6: Run the full publisher test module under the real target** + +A format-changing edit to this file can break tests far from the one you added, and the +Viceroy runner aborts on the first panic — so run the whole suite, not a filtered subset. + +```bash +cargo test-fastly +``` + +Expected: PASS. `app::tests` DNS `Error` lines in the output are pre-existing noise. + +- [ ] **Step 7: Verify format and lint** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly +``` + +Expected: both clean. + +- [ ] **Step 8: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-core/src/settings.rs \ + crates/trusted-server-core/src/test_support.rs +git commit -m "Add an operator switch for the origin cache bypass and log origin fetch time" +``` + +Staging without `settings.rs` leaves a tree that does not compile. + +- [ ] **Step 9: Deploy and collect** + +Deploy first. Then enable the log — it is gated and off by default: + +```bash +# In the operator's trusted-server.toml, under [debug]: +# publisher_timing = true +ts config push +``` + +**Deploy before pushing, not after.** `Settings`, `Publisher`, and `DebugConfig` all carry +`#[serde(deny_unknown_fields)]`, and `ts config push` validates against the typed schema +(`crates/trusted-server-cli` → `run_config_push_typed::`). So the +`ts` binary must be rebuilt from this commit (`cargo install-cli`), and pushing the new +keys before the new WASM is live would break config load on the deployed build. +`trusted-server.example.toml:121-125` records this same hazard for +`auction.rewrite_creatives`. + +Then capture the **bypass-on baseline only**. Do not try to collect an off arm here — +turning the bypass off _is_ Task 5, which is gated on Task 1's verdict and forbidden on a +FAIL. The off arm is collected in Task 5 Step 8. + +Capture enough navigations to separate the medians with confidence, across both a homepage and an article path, with the bypass +both on and off. Record the N alongside the result. + +Append to the findings document: + +```markdown +## Step C — server-side latency breakdown + +**N per arm:** · **Paths:** · **Date:** + +| Arm | `origin_fetch_ms` = origin TTFB (median) | auction `total_time_ms` (median) | `rewrite_ms` (median) | +| ---------- | ---------------------------------------- | -------------------------------- | --------------------- | +| bypass on | | | | +| bypass off | | | | + +Read the asymmetry carefully. `origin_fetch_ms` is origin **TTFB** — the send returns at +response headers because `.with_stream_response()` is set — whereas `total_time_ms` is +the auction's full duration. The comparison below is still the right one, but it is not +comparing two like quantities. + +**Verdict:** HOLD IS FREE / HOLD IS COSTING + +Read it off `hold_wait_ms` directly — no model, no comparison against auction duration. + +HOLD IS FREE = `hold_wait_ms` median near zero. The auction finishes during body +transfer. Proceed as staged in spec §7: Stage 0 primary, Stage 2 protects its win. + +HOLD IS COSTING = `hold_wait_ms` median materially non-zero. **Staging inverts** — +Stage 2 becomes primary and Stage 0 secondary. The work does not change, only its order. +Spec §6.2 argues for the first outcome but explicitly does not prove it, so treat the +second as a live possibility. +``` + +- [ ] **Step 10: Commit the findings** + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record server-side latency breakdown for #1009" +``` + +--- + +> **Task 4 (spec correction) was completed while this plan was being written.** §3's +> mechanism bullet, §4's operator-flag framing, and the `unexpected_origin_304` watch are +> all already in the spec. Nothing to do; the task is removed rather than left as a +> no-op an implementer would stall on. + +--- + +## Task 5: Stage 0 — turn the origin cache bypass off + +**Gate:** do not flip the flag until Task 1 has a recorded verdict. + +- **PASS** → Task 5a (config flip). +- **FAIL** → Task 5b. Do **not** flip on a FAIL; it can serve an RSC payload to an HTML + navigation. + +Because Task 3 made the bypass config-driven, Stage 0 ships as a **config change on an +already-deployed build**. No second release, and rollback is another config push rather +than a revert. That matters here specifically: the failure mode this gates on is cache +poisoning, where minutes of exposure are worse than a slow rollout. + +### Task 5a: flip the flag (Task 1 verdict = PASS) + +**Files:** + +- Modify: the operator's `trusted-server.toml` (gitignored) +- Modify: `crates/trusted-server-core/src/publisher.rs` — the test, and later the default +- Modify: `trusted-server.example.toml` — document the key + +- [ ] **Step 1: Add a test covering the flag in both positions** + +The existing test at `publisher.rs:4824` +(`eligible_navigation_bypasses_cache_and_returns_non_storable_html`) asserts `vec![true]` +and must **keep passing** while the default is `true` — it now documents the default +rather than the only behaviour. Leave it, and add a sibling next to it: + +```rust +#[tokio::test] +async fn eligible_navigation_uses_read_through_cache_when_bypass_disabled() { + // Arrange + let mut settings = settings_with_enabled_auction_and_creative_opportunities(); + settings.publisher.bypass_origin_cache = false; + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let response = + run_with_slots(&settings, &services, &slots, conditional_navigation_request()).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "disabling bypass_origin_cache should let the navigation use the read-through \ + cache; the conditional-header strip already guarantees a full body on a HIT" + ); + assert_eq!( + recorded_header( + stub.recorded_request_headers().first().expect("should record request"), + header::IF_NONE_MATCH.as_str() + ), + None, + "conditional headers must still be stripped with the bypass disabled" + ); + assert!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("no-store")), + "the synthesized document must stay non-storable regardless of the bypass flag" + ); +} +``` + +Those last two assertions are the point of the test: the flag must change **only** the +cache mode, leaving the conditional-header strip and the response non-storability intact. + +**Leave `publisher.rs:4941` and `:5160` unchanged** — they already assert `vec![false]` +for non-eligible requests and must keep doing so. `Range`/`If-Range` stripping is covered +by `eligible_range_navigation_fetches_complete_html` (`publisher.rs:4883`), unaffected. + +- [ ] **Step 2: Run both tests** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation -- --nocapture +``` + +Expected: both the existing default-behaviour test and the new flag-disabled test PASS. +If Task 3's `bypass_origin_cache` field is not yet in place, the new test will not +compile — land Task 3 first. + +- [ ] **Step 3: Document both keys in the example config** + +Add to `trusted-server.example.toml` under `[debug]` (line 149, alongside +`ja4_endpoint_enabled` and `auction_html_comment`): + +```toml +# Emit a `publisher_timing` log line per publisher origin fetch. Temporary +# instrumentation for the #1009 latency measurement; leave false in production. +publisher_timing = false +``` + +And under `[publisher]`: + +```toml +# Bypass the platform read-through cache on ad-eligible navigations. +# `true` is the historical default. Set `false` to let those navigations use the +# read-through cache — only after confirming the origin declares `Vary` for every +# header it varies on (see the Stage 0 precondition). +bypass_origin_cache = true +``` + +- [ ] **Step 4: Flip it in the operator config and push** + +```bash +# In the operator's trusted-server.toml, under [publisher]: +# bypass_origin_cache = false +ts config push +``` + +Note from prior operational experience in this repo: the environment-variable overlay is +scalar-only **and** only overrides keys that already exist in the TOML. Adding the key to +the operator's file is required; setting only an env var will be silently dropped. + +**Roll back by pushing `true` again.** No release required. That is the whole reason this +is a flag. + +- [ ] **Step 5: Run the full suite across every adapter** + +```bash +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +``` + +Expected: all PASS. If `platform/test_support.rs:797` or `:888` fail, they are testing +the stub's own recording behaviour rather than publisher behaviour — read them before +changing anything. + +- [ ] **Step 6: Format and lint every target** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare \ + && cargo clippy-cloudflare-wasm && cargo clippy-spin-native && cargo clippy-spin-wasm +``` + +Expected: all clean. + +- [ ] **Step 7: Commit the code and config-template changes** + +```bash +git add crates/trusted-server-core/src/publisher.rs trusted-server.example.toml +git commit -m "Add an operator switch for the publisher origin cache bypass" +``` + +- [ ] **Step 8: Watch for the failure modes, not just the win** + +After the flip, check three things before declaring success. The first two are regression +signals, not confirmations. + +1. **`unexpected_origin_304` abandonment telemetry.** This reason + (`publisher.rs:2896`, emitted via `emit_abandoned_auction` at `:2360`) exists because + the ad-stack path refuses cached and conditional origin responses. Re-enabling the + cache is precisely what could revive it. **Any non-zero rate is a rollback signal** — + it means a 304 is reaching TS, which the conditional-header strip was supposed to make + impossible. Push `true` and investigate before continuing. +2. **Representation mixing.** Spot-check that HTML navigations still return HTML and RSC + fetches still return `text/x-component`. A mismatch is the Task 1 risk having + materialized despite a PASS verdict — roll back immediately, this is cache poisoning. +3. **`origin_fetch_ms` and `cache_bypass=false`** in the `publisher_timing` logs. This is + the win, and it is the _last_ thing to check, not the first. + +- [ ] **Step 9: Record and commit** + +Append the before/after medians and the three checks above to the findings document, +format it, and commit. + +- [ ] **Step 10: Retire the flag (follow-up, not now)** + +Once the flip has held for a sustained period, flip the default to `false` in +`default_bypass_origin_cache`, then remove the setting and the branch entirely. Track it; +a temporary flag left in place becomes permanent configuration surface. + +### Task 5b: cache-key discriminator (Task 1 verdict = FAIL) + +**Do not implement from this plan.** A FAIL means the origin serves multiple +representations at one URL without declaring `Vary`, so removing the bypass requires TS +to add its own cache-key discriminator — a feature, not a deletion, and materially larger +than Stage 0 as scoped here. + +Escalate with the Task 1 findings and write a separate plan. Two things that plan must +address, both from spec §4: + +1. The discriminator must key on the request headers that actually distinguish the + representations (`RSC`, `Next-Router-*`, the experiment header), **not** on the + navigation classification. `is_navigation_request` + (`crates/trusted-server-core/src/http_util.rs:73-98`) falls back to the `Accept` + header when Fetch Metadata is absent, and its own comment warns that `fetch()` can set + `Accept: text/html` — so a fetch-based request can be misclassified as a navigation. +2. Whether the origin should simply be asked to declare `Vary`, which is cheaper than + building the discriminator and fixes the problem for every consumer rather than only + for TS. + +--- + +## Out of scope + +Named so nobody widens this plan mid-flight. All are specified in the spec. + +- **Stages 1–2** — moving bid delivery off the response body and deleting the `` + hold. Spec §7 and §8 put these behind the correctness defects. Spec §5 explains why + starting them casually produces a silent revenue loss. +- **Stages 3a/3b** — response cacheability. 3b is additionally gated on Task 2. +- **Stages 4–5** — purge capability, TS-owned template cache, ESI. +- **Removing the `bypass_cache` platform capability.** Task 5a removes one call site only. + +--- + +## Definition of done + +- [ ] Findings document records verdicts for Steps A, B, and C, each with its date, its + N where applicable, and the consequence spelled out. +- [ ] `publisher_timing` and `publisher_hold` lines are emitted in production and + readable, and `hold_wait_ms` has a recorded median. +- [ ] Either Task 5a is shipped, or Task 1 returned FAIL and both a production defect and + a follow-up plan for 5b exist. +- [ ] **The win is measured client-side, not from `origin_fetch_ms`.** That figure is + origin TTFB and excludes body download, rewrite, and post-processing — it is + attribution, not the outcome. #1009 already has a working tester-cookie browser A/B + measuring the TTFB the publisher actually complained about; use it for before/after. +- [ ] `unexpected_origin_304` rate is zero and representations are not mixed (Task 5a + Step 8) — both checked **before** the win is claimed. +- [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four + adapter test suites; the parity suite; JS build, test, and format; docs format. From 8e124ca0ade68ed47c5fc6a76404f13c390c5714 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 8 Aug 2026 15:49:53 +0530 Subject: [PATCH 168/395] Record Step A findings for #1009: origin Vary check passes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran the Stage 0 gate against the publisher origin. Verdict is PASS, so Stage 0 takes the operator-flag path rather than the cache-key discriminator, and there is no live cross-serving defect. The origin declares vary on rsc, next-router-state-tree, next-router-prefetch, next-router-segment-prefetch and Accept-Encoding, covering every header that distinguishes the HTML and RSC representations sharing a URL. It names one the plan did not think to probe. Bodies do not differ by cookie, no Set-Cookie rides a shared-cacheable response, and the origin answers without credentials. Two things the check was not looking for. The origin already sets cache-control: max-age=60 with a correct Vary, so it has been cacheable all along and Trusted Server opted out of it — though a 60 second TTL also bounds the win. And the document regenerates roughly 170 ad-slot container IDs as fresh UUIDs per request, so a cached copy serves identical IDs to every visitor within the TTL. That is probably harmless because slot definitions come from config rather than origin markup, but it is an untested interaction with slot matching and belongs on the pre-flip checklist. Also fixes a defect in the plan's own probe. It compared body hashes, which on this origin differ on every request because of those UUIDs, cookie or not — it would have reported a false FAIL every time. Replaced with normalize-then-diff against a measured no-cookie baseline, and noted that the Host override is required because the origin is a shared vhost. --- ...2026-08-08-1009-measurement-and-stage-0.md | 35 ++++- .../2026-08-08-1009-measurement-findings.md | 121 ++++++++++++++++++ 2 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-08-1009-measurement-findings.md diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md index c5f85aa4c..a28da9186 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md @@ -162,17 +162,38 @@ class: TS forwards client cookies to origin unchanged, so any cookie-personalize (logged-in state, paywall meter, publisher-side A/B assignment) becomes cross-servable once the cache is on. +**Do not compare body hashes.** Verified on the live origin: this page regenerates +~170 ad-slot container IDs as fresh 32-hex UUIDs on every request, so three requests give +three different hashes with byte-identical lengths, cookie or not. A hash comparison +reports a false FAIL every time. + +Normalize per-request identifiers, establish the no-cookie baseline drift first, then ask +whether the cookie arm differs by _more_ than that baseline: + ```bash -# Same URL, with and without a session cookie. Compare Content-Length and body hash. -for C in '' 'Cookie: '; do - echo "--- ${C:-no-cookie}" - curl -sS -D /dev/stderr -o - "https:///" \ - -H 'Sec-Fetch-Dest: document' -H 'Accept: text/html' ${C:+-H "$C"} \ - 2> >(grep -iE '^(vary|cache-control|set-cookie|content-length):' >&2) \ - | shasum +ORIGIN="https://"; HOSTH="Host: " +norm() { sed -E 's/[0-9a-f]{32}/UUID/g' "$1"; } + +for n in a b; do + curl -sS "$ORIGIN/" -H "$HOSTH" \ + -H 'Sec-Fetch-Dest: document' -H 'Accept: text/html' > "nc_$n.html" done +curl -sS "$ORIGIN/" -H "$HOSTH" \ + -H 'Sec-Fetch-Dest: document' -H 'Accept: text/html' \ + -H 'Cookie: ' > ck.html + +echo "baseline drift: $(diff <(norm nc_a.html) <(norm nc_b.html) | grep -c '^[<>]')" +echo "with cookie: $(diff <(norm nc_a.html) <(norm ck.html) | grep -c '^[<>]')" +diff <(norm nc_a.html) <(norm ck.html) | head -20 ``` +Send the `Host` override — the origin is a shared vhost and will not return the right +document without it. Read it from `publisher.origin_host_header_override`. + +**Step A has already been run and passed.** See +[the findings](./2026-08-08-1009-measurement-findings.md). Re-run only if the origin +changes. + Three failure shapes, any of which blocks Stage 0 independently of the `Vary` verdict: - Bodies differ by cookie **and** `Vary` does not name `Cookie` → cross-serving of diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md new file mode 100644 index 000000000..78f0a3d78 --- /dev/null +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -0,0 +1,121 @@ +# #1009 measurement findings + +Recorded output of the checks in +[the plan](./2026-08-08-1009-measurement-and-stage-0.md). Results only — the origin +hostname is operator config and is deliberately not reproduced here. + +## Step A — origin `Vary` declaration and cookie exposure + +**Date:** 2026-08-08 · **Method:** direct `curl` against the publisher origin with the +configured `origin_host_header_override`, homepage path. + +### Representation split + +| Representation | `Content-Type` | `Cache-Control` | `Set-Cookie` | +| ------------------------------------ | ------------------ | --------------- | ------------ | +| HTML navigation | `text/html` | `max-age=60` | none | +| `RSC: 1` | `text/x-component` | `max-age=60` | none | +| `RSC: 1` + `Next-Router-Prefetch: 1` | `text/x-component` | `max-age=60` | none | + +`Vary`, identical on every response: + +``` +vary: rsc, next-router-state-tree, next-router-prefetch, next-router-segment-prefetch, Accept-Encoding +``` + +The origin declares **every** header that distinguishes the representations, including +`next-router-segment-prefetch`, which the plan's probe list did not think to check. The +HTML/RSC split at one URL is real and correctly declared. + +### Cookie personalization + +Hash comparison was useless here and the plan's probe as written would have produced a +false FAIL — see the method note below. After normalizing per-request identifiers: + +| Comparison | Differing lines | +| ------------------------------ | --------------- | +| no-cookie A vs no-cookie B | 2 | +| no-cookie A vs **with cookie** | 2 | + +Both diffs are the same single `generationTimestamp` field in the RSC payload. **The +cookie changes nothing.** Byte lengths were identical across all three responses +(1,432,944). + +Cookie sent: `ts-tester=true; sessionid=abc123; ts-ec=probe`. + +### Verdict: **PASS** + +- `Vary` names every request header the origin varies on. ✅ +- Bodies do not differ by cookie, so `Vary: Cookie` is not required. ✅ +- No `Set-Cookie` on a shared-cacheable response. ✅ +- Origin returns 200 without credentials, so no `Authorization` exposure at this layer. + (#1009's basic-auth gate is on the Trusted Server side, not the origin.) ✅ + +**Consequence:** Stage 0 takes the simple path — the operator flag plus a config flip, +not the cache-key discriminator. No live production defect: the origin's `Vary` covers +the RSC variants that already transit the read-through cache today. + +## Two findings the checks were not looking for + +### 1. The origin already intends this page to be shared-cached + +`cache-control: max-age=60` with a correct `Vary` and no `Set-Cookie`. The origin has +been cacheable all along; Trusted Server opted out of it. That is the spec's §4 framing +confirmed from the other side, and it strengthens the case that the bypass was +belt-and-braces rather than load-bearing. + +It also bounds the win: a 60-second TTL means Stage 0 buys a cache hit only within that +window. Whether that translates into a meaningful hit rate depends on request volume per +URL, which is not measured here. + +### 2. Ad-slot div IDs are randomized per request — and this interacts with Stage 0 + +The only per-request variance in the document is ~170 lines of ad-slot container IDs, +each a fresh 32-hex UUID: + +``` +ad-in_content-f75fa7fba54a4fc2a2d787f51c1837dd-in_content-0 +ad-in_content-a968b27e3ee2424f8bb1c19560abf2b1-in_content-0 ← same slot, next request +``` + +Under the bypass, Trusted Server sees fresh IDs on every request. **Once the cache is on, +every visitor within a 60-second window receives the same IDs.** + +This is very likely fine — `tsjs.adSlots` is built from configured slot definitions, not +scraped from origin markup, and injection is a prefix match on the configured `div_id`. +But it is an untested interaction between Stage 0 and the slot-matching path, and it was +not in anyone's risk list. **Verify slot matching still resolves against a cached +document before flipping the flag**, and watch TS-attributed renders across the flip +rather than only `origin_fetch_ms`. + +## Method note — a defect in the plan's Step A probe + +The plan's cookie check compares `shasum` of the response bodies. On this origin that +test always fails, cookie or not, because of the randomized div IDs above. Three requests +produced three different hashes with byte-identical lengths. + +**Correct method:** normalize per-request identifiers before comparing, e.g. +`sed -E 's/[0-9a-f]{32}/UUID/g'`, and diff the normalized bodies rather than hashing +them. Establish the no-cookie baseline drift first, then compare the cookie arm against +that baseline — a cookie arm is only interesting if it differs by _more_ than the +baseline does. Fix the plan before anyone re-runs this. + +## Residual uncertainty + +#1009 states the origin varies on an experiment header as well as `rsc` and +`next-router-*`. **No experiment header appears in the origin's `Vary` list**, and the +RSC payload's `experiments` key did not differ across any of the requests made here. + +Three readings, unresolved: the issue was imprecise; experiments are assigned +client-side; or they key on a cookie value this probe did not supply. The `Vary` +declaration is authoritative for cache correctness and it is thorough enough to name four +Next-specific headers, so this is unlikely to be a cache-safety gap. Worth one question +to whoever wrote that line in #1009 rather than further probing. + +## Step B — consumers of TS's own response headers + +Not yet run. + +## Step C — hold and origin fetch timings + +Not yet run. From 86fa20f6395af43499aa1bcf67e69d1bf2638fa3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 14:51:36 +0530 Subject: [PATCH 169/395] Reopen ESI as feasible and separate it from the Stage 0 optimisation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An external review rejected the previous revision's central conclusion and was right to. Verified against the pinned fastly 0.12.1. ESI was called structurally blocked on two grounds, both false. The cache boundary it needs is native: cache::core provides insert(key, max_age).execute() returning a StreamingBody for arbitrary bytes, lookup()/found() to read them back, and Transaction with must_insert() for request collapsing. No separate KV or template service is required. And purge exists in-process via InsertBuilder::surrogate_keys plus http::purge::purge_surrogate_key, so the management-API token scope previously cited is the wrong surface entirely. The error was inspecting what this repository does and reporting it as what the platform permits, which is the same mistake the document criticises #1009 for making in the other direction. The correction is recorded at the top of the spec rather than quietly edited in. The pipeline ordering was also backwards. It said order esi then lol_html; lol_html is what emits the esi:include tags, so ESI must run after it. New section 6.6 gives the corrected pipeline and separates the three caches the documents had been conflating: origin read-through, shared transformed template, and a final assembled-response cache that must never exist. #418 is React's error number, not a repository issue. The tracker is #938. Stage 0 is reframed as a supporting optimisation and the experimental control, not an answer to #1009 — it has no ESI or client-fill arm, so completing it cannot close the issue. Its rollback claim is corrected: flipping the flag stops HTML reading from cache but evicts nothing, so rollback needs a purge or a versioned key namespace and observation past the origin TTL. Step A is downgraded from PASS to provisional. It used a synthetic session cookie, one route, no experiment variant, and no authenticated session through TS. Cached-hit slot resolution becomes a release gate rather than a note. Adds the ESI validation spike plan: four comparable arms plus a TS-off reference, a deterministic synthetic fragment before the real auction, safety gates run against every arm rather than once at the end, a decision rule ratified before collection, purge-based rollback, and reproducibility metadata. --- ...2026-08-08-1009-measurement-and-stage-0.md | 29 +- .../2026-08-08-1009-measurement-findings.md | 50 +- .../2026-08-10-1009-esi-validation-spike.md | 460 ++++++++++++++++++ ...08-esi-cacheable-root-validation-design.md | 247 +++++++--- 4 files changed, 694 insertions(+), 92 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md index a28da9186..3aa100ade 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md @@ -2,9 +2,16 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Obtain the checks that gate the #1009 work, then turn off the redundant origin -cache bypass that the spec identifies as the actual TTFB cost — behind an operator flag, -so it rolls back with a config push rather than a release. +**Goal:** Turn off the redundant origin cache bypass that the spec identifies as the +actual TTFB cost, behind an operator flag, and establish the measurement baseline that +later work is compared against. + +> **This plan does not close #1009.** It contains no ESI arm and no client-fill arm, so +> completing it cannot answer whether ESI separates cacheable content from per-user +> state. It is a **supporting optimisation and the experimental control** for +> [the ESI validation spike](./2026-08-10-1009-esi-validation-spike.md), which is where +> #1009 is actually decided. Scoped and framed this way after external review on +> 2026-08-10. **Architecture:** Two investigation tasks that produce recorded findings and no code; one code task that adds a config-gated timing log and makes the cache bypass operator- @@ -864,8 +871,20 @@ Note from prior operational experience in this repo: the environment-variable ov scalar-only **and** only overrides keys that already exist in the TOML. Adding the key to the operator's file is required; setting only an env var will be silently dropped. -**Roll back by pushing `true` again.** No release required. That is the whole reason this -is a flag. +**Rollback is a config push plus an eviction — not a config push alone.** Pushing `true` +again stops HTML navigations reading from cache, but evicts nothing: objects already +cached, including those RSC and other request classes keep reading, persist until they +expire. The origin's `max-age=60` bounds that, but does not remove it. + +Full rollback: + +1. Push `bypass_origin_cache = true`. +2. Purge. `fastly::http::purge::purge_surrogate_key` runs inside Compute, with keys + attached at insert via `InsertBuilder::surrogate_keys`; alternatively roll a versioned + cache-key namespace. **Neither is wired today** — if the flip ships before one exists, + the rollback story is "wait out the TTL," and that must be an accepted risk rather + than an unnoticed one. +3. Observe past the origin TTL before declaring the incident closed. - [ ] **Step 5: Run the full suite across every adapter** diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md index 78f0a3d78..a125d957c 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -43,17 +43,32 @@ cookie changes nothing.** Byte lengths were identical across all three responses Cookie sent: `ts-tester=true; sessionid=abc123; ts-ec=probe`. -### Verdict: **PASS** +### Verdict: **PROVISIONAL PASS** — not sufficient to gate a production flip + +Downgraded 2026-08-10 after external review. Everything below held under the conditions +tested; the conditions tested are narrower than the gate requires. + +What passed: - `Vary` names every request header the origin varies on. ✅ -- Bodies do not differ by cookie, so `Vary: Cookie` is not required. ✅ +- Bodies did not differ by the cookie sent, so `Vary: Cookie` was not required **for + that cookie**. ✅ - No `Set-Cookie` on a shared-cacheable response. ✅ -- Origin returns 200 without credentials, so no `Authorization` exposure at this layer. - (#1009's basic-auth gate is on the Trusted Server side, not the origin.) ✅ +- Origin returns 200 without credentials at this layer. ✅ + +**What was not tested, and each of these can flip the verdict:** -**Consequence:** Stage 0 takes the simple path — the operator flag plus a config flip, -not the cache-key discriminator. No live production defect: the origin's `Vary` covers -the RSC variants that already transit the read-through cache today. +| Gap | Why it matters | +| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `sessionid=abc123` is not a real session | A synthetic value proves nothing about a state-bearing publisher session. An authenticated or paywall-metered session is exactly the case that would personalize. | +| One route (homepage) only | Article, section, and search routes may personalize differently. | +| Experiment variant never exercised | #1009 says the origin varies on one. It is absent from `Vary` — see Residual uncertainty below. | +| Basic Auth through TS untested | #1009 describes a gated deployment. Only the origin was probed directly. | +| Cached-hit slot resolution untested | The randomized div IDs below are an unverified interaction, not a cleared one. | + +**Consequence:** Stage 0 still takes the operator-flag path rather than the cache-key +discriminator, and no live cross-serving defect is indicated. But this is **not** a +release gate. Close the table above before flipping the flag in production. ## Two findings the checks were not looking for @@ -84,8 +99,8 @@ every visitor within a 60-second window receives the same IDs.** This is very likely fine — `tsjs.adSlots` is built from configured slot definitions, not scraped from origin markup, and injection is a prefix match on the configured `div_id`. But it is an untested interaction between Stage 0 and the slot-matching path, and it was -not in anyone's risk list. **Verify slot matching still resolves against a cached -document before flipping the flag**, and watch TS-attributed renders across the flip +not in anyone's risk list. **This is a release gate, not a note.** Verify slot matching resolves against a cached +document before flipping the flag, and watch TS-attributed renders across the flip rather than only `origin_fetch_ms`. ## Method note — a defect in the plan's Step A probe @@ -112,6 +127,23 @@ declaration is authoritative for cache correctness and it is thorough enough to Next-specific headers, so this is unlikely to be a cache-safety gap. Worth one question to whoever wrote that line in #1009 rather than further probing. +## Rollback caveat, added 2026-08-10 + +The plan described flipping the flag back as a seconds-long rollback. That is +incomplete. Re-enabling the bypass stops **HTML navigations** reading from cache; it +evicts nothing. Objects already cached — including those RSC and other request classes +continue to read — persist until they expire. + +Two mitigations, both real: + +- The origin's `max-age=60` bounds read-through exposure to roughly a minute. +- Purge is available in-process: `fastly::http::purge::purge_surrogate_key`, with keys + attached at insert via `InsertBuilder::surrogate_keys`. An earlier claim that TS had no + purge capability was wrong — it has no _wiring_, which is buildable. + +Rollback is therefore: flip the flag, **then** purge or roll a versioned cache-key +namespace, **then** observe past the origin TTL before declaring the incident closed. + ## Step B — consumers of TS's own response headers Not yet run. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md new file mode 100644 index 000000000..bbfd7de84 --- /dev/null +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -0,0 +1,460 @@ +# #1009 ESI Validation Spike + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps +> use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Decide #1009 on evidence. Build a shared-template pipeline behind a flag, run +ESI and client-fill against it, and produce a decision record that either adopts ESI, +adopts client-fill, or rejects both — with the Fastly-only maintenance cost priced in. + +**Architecture:** `origin → lol_html transform → fastly::cache::core → assemble → finalize`. +The transform emits `esi:include` markers at the two existing injection seams instead of +inlining per-user data. The cached object is a shared template with no per-user bytes. +Assembly is either the `esi` crate (edge) or a client fetch of `/_ts/page-bids` (browser), +selected per request by config so both can be measured on one build. + +**Tech Stack:** Rust 2024, `wasm32-wasip1`, `fastly` 0.12.1 (`cache::core`, `http::purge`), +`esi` 0.7, `lol_html`, a real Fastly test service for cache behaviour. + +**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` — +read the 2026-08-10 correction at the top and +[§6.6](../specs/2026-08-08-esi-cacheable-root-validation-design.md#66-the-esi-pipeline-corrected) +before writing any code. + +**Control:** [the Stage 0 plan](./2026-08-08-1009-measurement-and-stage-0.md). Its +instrumentation and its bypass flag are prerequisites — this plan compares against them +and does not duplicate them. + +--- + +## Why this plan exists + +An earlier revision of the spec concluded ESI was structurally impossible. It was wrong: +`fastly::cache::core` provides the cache boundary natively, and purge runs inside Compute. +That correction reopens #1009 as an empirical question, and this plan is how it gets +answered. + +**What is genuinely uncertain**, and what each arm is for: + +1. Does a shared template plus per-request assembly beat today's inline path enough to + matter? +2. Does **edge** assembly (ESI) beat **client** assembly (a fetch) by enough to justify a + Fastly-only rendering path that must be maintained alongside the portable one? +3. Can per-user leakage be excluded across cold MISS, warm HIT, stale revalidation, + transform failure, and fragment failure? + +Question 3 is a gate, not a metric. A win on 1 and 2 with a failure on 3 is a rejection. + +## Three caches, never conflated + +The original error came from treating these as one thing. Every task below names which it +means. + +| # | Cache | Contents | Status | +| --- | --------------------------------- | ----------------------------- | ----------------------------------- | +| C1 | Origin read-through | raw origin bytes | Exists. Stage 0 turns it back on. | +| C2 | Shared transformed template | post-`lol_html`, pre-assembly | **New.** What this plan builds. | +| C3 | Assembled-response delivery cache | final per-user output | **Must never exist.** Not proposed. | + +If a task appears to require C3, stop — that is the leakage failure mode, not a design +option. + +## Arms + +Five, but only four are treatable as equivalent. + +| Arm | Root | Bids | Notes | +| ------- | ----------------------- | ---------------- | ---------------------------------------------------------- | +| **A0** | inline, C1 bypassed | inline `` | Today. The baseline. | +| **A1** | inline, C1 on | inline `` | Stage 0. Isolates the bypass from the template change. | +| **A2** | shared template from C2 | client fetch | Portable. Works on all four adapters. | +| **A3** | shared template from C2 | ESI at the edge | Fastly-only. The thing #1009 proposed. | +| **REF** | origin direct, TS off | publisher's own | **Reference, not an arm.** Different work, not comparable. | + +A0→A1 measures the bypass. A1→A2 measures the template split. A2→A3 measures edge versus +client assembly — **that difference is the entire case for ESI**, and it is the number +this plan exists to produce. + +REF is included because #1009 anchors on it, and excluded from pass/fail because TS-off +does no auction and no injection. Comparing against it measures the feature's existence, +not its implementation. + +--- + +## Task order and dependencies + +``` +Stage 0 plan (flag + timing instrumentation) ──┐ + ├──> Task 3 (C2 template cache) +Task 1 (esi crate compiles) ───────────────────┤ +Task 2 (test service + harness) ────────────────┘ │ + ├──> Task 4 (A2 client-fill) + ├──> Task 5 (A3 ESI) + └──> Task 6 (safety gates) + │ + └──> Task 7 (decision record) +``` + +Tasks 1 and 2 are independent and should run first — both can invalidate the plan +cheaply. Task 6 runs against every arm, not once at the end. + +--- + +## Task 1: Confirm `esi` 0.7 builds on this toolchain + +Cheapest possible falsification. Do this before anything else. + +**Files:** `crates/trusted-server-adapter-fastly/Cargo.toml` + +- [ ] **Step 1: Add the dependency** + +```bash +cargo add esi@0.7 --package trusted-server-adapter-fastly +``` + +It belongs in the **Fastly adapter**, never in `trusted-server-core` — the crate is +hard-bound to `fastly::{Request, Response, Backend}` and core must stay portable. + +- [ ] **Step 2: Check it compiles for the real target** + +```bash +cargo check-fastly +``` + +Expected: clean. The crate declares edition 2021 with no `rust-version`, and pulls recent +`rand` and `nom`, so this is a genuine question on Rust 1.95.0 / `wasm32-wasip1`. + +- [ ] **Step 3: Check the lockfiles have not desynced** + +```bash +git diff --stat Cargo.lock +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +``` + +CI requires shared direct deps to match between the root and integration-tests lockfiles. +`regex`, `bytes`, and `log` overlap. If they desync, fix with targeted +`cargo update -p --precise ` — **never a full update**. + +- [ ] **Step 4: Record and commit, or stop** + +If Step 2 fails, this plan stops here and #1009 is answered "not on this toolchain." +Record that in the findings document and escalate rather than fighting the build. + +```bash +git add crates/trusted-server-adapter-fastly/Cargo.toml Cargo.lock +git commit -m "Add the esi crate to the Fastly adapter for the #1009 validation spike" +``` + +--- + +## Task 2: Stand up the test service and the harness + +**Viceroy 0.17 cannot exercise the `cache::core` hooks end to end.** Unit tests cover the +transform and the security properties; MISS / HIT / stale / shielding must run on a real +Fastly service. Establish that before building, or Tasks 3–6 have nowhere to run. + +- [ ] **Step 1: Provision a dedicated test service** + +Separate from production. Confirm and record: whether the publisher backend is +**shielded**, and whether any Delivery service fronts the Compute service. Both change +what the numbers mean. + +```bash +fastly service list +fastly backend list --service-id --version latest +``` + +The shielding answer also settles an open question from the Stage 0 findings: #1009's +off-TS win came from a shield HIT, so whether the test service has one determines whether +its numbers transfer to production at all. + +- [ ] **Step 2: Extend the harness for correlation** + +The existing tester-cookie A/B has no way to join server timings to browser timings. Add +a per-request correlation ID — generated at TS entry, echoed in an `x-ts-request-id` +response header, and included in every timing log line. + +Without it, the experiment cannot join hold time, origin time, auction telemetry, browser +TTFB, and render outcome for the same request. **That is the difference between an +experiment and a pile of numbers.** + +- [ ] **Step 3: Capture cache tier and status per request** + +Record `x-cache`, `hit-state`, `age`, and the serving POP alongside each measurement. A +median that mixes cold-MISS and warm-HIT requests is meaningless, and arms cannot be +compared unless the mix is known. + +- [ ] **Step 4: Define the sample plan before collecting anything** + +State, in the findings document, ahead of time: requests per arm per route, how cold MISS +is forced, how warm HIT is confirmed, and the confidence interval to be reported. + +Rationale: this whole effort exists because #1009 drew a causal conclusion from N=4 that +did not survive contact with the code. Repeating that with more arms would be worse, not +better. + +--- + +## Task 3: Build C2 — the shared transformed-template cache + +The core of the spike. Behind a flag, default off. + +**Files:** + +- `crates/trusted-server-core/src/publisher.rs` — emit markers at the two seams +- `crates/trusted-server-core/src/settings.rs` — the mode flag +- `crates/trusted-server-adapter-fastly/src/` — the `cache::core` read/write + +- [ ] **Step 1: Add the assembly-mode setting** + +```rust +/// How per-user ad state reaches the page. +/// +/// `Inline` is today's behaviour: bids injected before ``, root uncacheable. +/// `ClientFill` and `Esi` both serve a shared template from the transformed-template +/// cache and fill the holes afterwards. Spike-only — remove with the spike. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AssemblyMode { + #[default] + Inline, + ClientFill, + Esi, +} +``` + +Default `Inline` so the flag is a no-op until set. Note the hazards the Stage 0 plan +already documents: `Settings` carries `#[serde(deny_unknown_fields)]`, `ts config push` is +typed, and `Publisher` has a hand-written `Default` plus eight exhaustive test literals +and a live doctest. + +- [ ] **Step 2: Emit markers instead of inlining, under `ClientFill`/`Esi`** + +The two seams are already isolated — that is #1009's correct observation. At head-open, +`tsjs.adSlots` is **per-URL and stays in the template** (config- and path-derived only, +`publisher.rs:3501-3525`). At body-close, emit a marker instead of the bids script. + +Under `Esi`: ``. +Under `ClientFill`: nothing at all — **not an empty bids script.** The Stage 0 plan +explains why: an empty script calls `scheduleInitialAdInit({})` and assigns +`ts.bids = {}` synchronously, racing the client fetch. + +**Assert the template carries no per-user bytes.** A unit test over the transform output +must fail on any of: a bid value, an EC ID, a consent string, a geo value, or a +`Set-Cookie`. This is the test that makes C2 safe, and it is cheaper to write now than to +retrofit. + +- [ ] **Step 3: Write the template into C2** + +```rust +// Fastly adapter. Key on the same signals the origin varies on, plus TS's own +// variant inputs. Surrogate-key it so rollback can purge rather than wait. +let mut insert = fastly::cache::core::insert(cache_key, template_ttl); +insert.surrogate_keys([&surrogate_key_for_url, "ts-template"]); +let mut body = insert.execute()?; +// stream the lol_html output into `body` +``` + +Use `cache::core::Transaction` with `must_insert()` for the lookup, so a cold cache under +load transforms once rather than per concurrent request. + +**Cache key must include** everything the origin's `Vary` names — `rsc`, +`next-router-state-tree`, `next-router-prefetch`, `next-router-segment-prefetch`, +`Accept-Encoding` (measured, see the Stage 0 findings) — **plus** TS's own per-variant +inputs: request host and scheme, the enabled-integration set, and the tsjs content hash. +Per-user signals must never appear in the key; they must be absent from the template +instead. If a signal cannot be excluded from the template, it does not belong in C2. + +Set `template_ttl` deliberately short for the spike. A short TTL bounds every failure mode +here and costs only hit rate. + +- [ ] **Step 4: Read it back and assemble** + +On `found()`, skip the origin fetch and the transform entirely; hand the cached body to +the assembler. On miss, transform and insert as above, then assemble from what was +inserted. + +- [ ] **Step 5: Unit tests, then the target suite** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin assembly_mode +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +cargo fmt --all -- --check && cargo clippy-fastly +``` + +`ClientFill` must work on all four adapters. `Esi` is Fastly-only and must not break the +others' compilation. + +--- + +## Task 4: Arm A2 — client-fill + +Mostly already specified. See +[the spec's Appendix B](../specs/2026-08-08-esi-cacheable-root-validation-design.md#appendix-b--stage-1-plumbing-condensed) +for the client plumbing, the two-condition join gate, and the server contract; and +[§5](../specs/2026-08-08-esi-cacheable-root-validation-design.md#5-the-trap-in-the-deferred-work--read-this-before-scheduling-stages-12) +for the silent-empty-bids trap, which applies in full. + +- [ ] **Step 1: Hoist the closure-trapped client state** — `pageBidsEndpoint`, + `requestPageBids`, and the `inflight`/`currentPath`/`lastAppliedPath` state, per + Appendix B. Do **not** route the initial load through `onNavigate`. +- [ ] **Step 2: Make `installScheduleInitialAdInit` a hydration-ready AND bids-settled + join**, with a bounded timeout that fires `adInit` untargeted rather than stranding + the slot. Derive the timeout from measured fetch latency, not a constant. +- [ ] **Step 3: Suppress the navigation-path dispatch** so exactly one auction runs per + pageview. Add a new `AuctionSource` for initial loads **plus the mechanism that + delivers it** — a header behind the same-origin gate, not a query parameter. +- [ ] **Step 4: Relocate terminal telemetry.** Navigation `Completed` is emitted only from + the collect functions; the `ts-debug` dump rides the same string. Both move. +- [ ] **Step 5: Verify exactly one auction per pageview** in `auction_events_raw`. Two is + a doubling of SSP spend and an immediate fail. + +--- + +## Task 5: Arm A3 — ESI at the edge + +- [ ] **Step 1: Wire `process_stream`, not the wrappers** + +`process_response` and `process_response_streaming` consume `self` _and_ send the response +themselves, which takes ownership away from the finalize / `ec_finalize` / apply-effects +ordering. `process_stream(&mut self, src: impl BufRead, out: &mut impl Write, …)` keeps it. + +Source is the C2 body. Sink is the response body on the way to the client — so EC cookie, +geo, and the privacy net still run **after** assembly. Confirm that ordering explicitly; +it is the difference between a correct response and a leaked one. + +- [ ] **Step 2: Disable DCA explicitly and allowlist the dispatcher** + +```rust +let config = esi::Configuration::default() + .with_escaped(false); +// default_dca and inherit_parent_dca stay at DcaMode::None / false — set them +// explicitly rather than relying on defaults; this is a pre-1.0 crate and the +// setting fails open. +``` + +The dispatcher must be **exact-path allowlisted**: a fragment URL that is not the bids +endpoint is refused, not fetched. The built-in dispatcher builds a dynamic backend per URL +host and panics on a hostless URL — never use it. + +Rationale in the spec's §2: bid payloads carry partner-controlled creative markup, so a +recursive parse would let an SSP make the edge fetch an arbitrary URL. **Add a unit test +that feeds `` through a creative payload and +asserts no fetch is attempted.** + +- [ ] **Step 3: Deterministic synthetic fragment first** + +Before the real auction, point the include at a fixed-content endpoint. This separates +"does the pipeline assemble correctly" from "does the auction behave," and the two fail +very differently. Only once assembly is proven does the include move to +`/_ts/page-bids`. + +- [ ] **Step 4: Handle the flush hazard** + +`esi` flushes its output writer after each parse batch. Fastly's `StreamingBody` is a +`BufWriter`, so anything between esi and it must propagate `flush()` or nothing leaves the +Wasm heap. + +- [ ] **Step 5: Fragment failure must degrade, not break** + +Assert that a fragment timeout or non-2xx yields a page with empty bids rather than a 5xx +or a truncated document. Note the crate's non-obvious semantics: `alt` is attempted before +`onerror="continue"`, and `` runs **all** attempts and concatenates every +non-failed output — it is not first-success-wins. + +--- + +## Task 6: Safety gates — run against every arm + +Not a phase. Every one of these is a hard fail, independent of any performance result. + +- [ ] **Zero cross-user leakage.** Request the same URL as two synthetic users differing + in consent state, EC identity, and geo. Assert the C2 template is byte-identical + and that no bid, EC ID, consent string, or geo value appears in it. +- [ ] **Cold MISS, warm HIT, stale revalidation** each produce a correct page. +- [ ] **Transform failure** (the 16 MB buffer cap, a malformed body) does not insert a + partial template into C2 and does not serve one. +- [ ] **Request collapsing** works: concurrent cold requests transform once. +- [ ] **DCA disabled**, verified by the injection test in Task 5 Step 2. +- [ ] **Exactly one auction per pageview**, from `auction_events_raw`. +- [ ] **Cookie and privacy finalization still run** after assembly — EC `Set-Cookie` on + first visit, and the privacy net downgrading it. This is the ordering that ESI's + streaming mode makes easy to get wrong, since it drops `$add_header`. +- [ ] **Slot and bid attribution unchanged.** Same slots matched, same bids applied, same + renders attributed. Use TS-attributed renders — the SSAT line item, non-empty + `ts.bids`, `hb_adid` presence — **never slot fill**, which is blind to empty bids + because `adInit` defines slots regardless. +- [ ] **No C3.** Assert the final assembled response is never shared-cacheable: no + `public`, no `s-maxage`, no `Surrogate-Control` on a response carrying per-user + state. + +--- + +## Task 7: The decision record + +**Files:** `docs/superpowers/plans/2026-08-10-1009-esi-decision-record.md` + +- [ ] **Step 1: Record every arm** with N, confidence interval, cache-tier mix, route mix, + and POP. Any arm missing those is not reportable. + +- [ ] **Step 2: Apply the decision rule, stated here before the data exists** + +**Adopt ESI only if all three hold:** + +1. Every Task 6 gate passes on A3. +2. A3 beats A2 on TTFB by a margin the reviewers ratify **before** collection — not + chosen after seeing the numbers. +3. Render outcomes on A3 are non-inferior to A0. + +**Otherwise adopt A2 (client-fill)** if its gates pass and it beats A1. It is portable +across all four adapters and carries no Fastly-only maintenance burden. + +**Otherwise keep A1** — Stage 0 alone — and record #1009 as answered in the negative with +evidence. + +The margin in (2) exists because A3's cost is not its diff. It is a second rendering +architecture, Fastly-only, on a pre-1.0 crate, in the critical render path. A small win +does not pay for that. + +- [ ] **Step 3: Record what would change the answer**, so this does not get re-litigated + from scratch. At minimum: React #418 / [#938](https://github.com/IABTechLab/trusted-server/issues/938) + being fixed such that `adInit` can run synchronously, which is what would make edge + assembly's round-trip saving actually worth something. + +- [ ] **Step 4: Clean up.** Remove the spike flag or promote it to a real setting; purge + C2 (`purge_surrogate_key` on `ts-template`); remove the synthetic fragment endpoint; + and either land or delete the `esi` dependency. **A spike flag left in place becomes + permanent configuration surface.** + +--- + +## Reproducibility metadata + +Record with every result, or it cannot be re-run or trusted: commit SHA; `esi` and +`fastly` crate versions; Fastly service and version IDs; whether the backend is shielded; +`template_ttl`; the origin's `Cache-Control` and `Vary` at collection time; assembly mode; +routes; N per arm; and the cache-tier mix. + +## Out of scope + +- **Stages 1–2 of the spec** as production work. This spike may build parts of the + client-fill path to measure it; shipping it is a separate decision behind the + correctness defects. +- **Full RSC/flight partitioning.** `rsc_flight.rs` has no static/dynamic split. +- **Publisher-authored ESI.** Breaks the no-origin-changes promise. +- **A C3 delivery cache.** Not a deferred item — a thing that must not exist. + +## Definition of done + +- [ ] Task 1 verdict recorded: `esi` 0.7 builds on Rust 1.95.0 / `wasm32-wasip1`, or it + does not and the spike stopped. +- [ ] All four arms measured on one build, with correlation IDs joining server and browser + timings, and cache tier recorded per request. +- [ ] Every Task 6 gate has an explicit pass/fail per arm. +- [ ] Decision record exists, applies the pre-ratified rule, and names what would change + the answer. +- [ ] Cleanup complete: flag resolved, C2 purged, synthetic endpoint removed, dependency + landed or dropped. +- [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four + adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md index a2e23b4f9..1417c5122 100644 --- a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -1,17 +1,45 @@ # ESI and the Cacheable Root -**Issue:** IABTechLab/trusted-server#1009 · **Date:** 2026-08-08 -**Baseline:** citations verified at `cfb98f4`; unchanged as of `b0ce56c3` (the two -commits between touch only CI workflows and Cargo aliases). +**Issue:** IABTechLab/trusted-server#1009 · **Date:** 2026-08-08 · +**Revised:** 2026-08-10 +**Baseline:** citations verified at `cfb98f4`; unchanged as of `b0ce56c3`. -**Decision requested:** approve the four items below. Three are "yes/no"; one funds -about three days of measurement. +> ## ⚠️ Correction, 2026-08-10 — this document's original ESI verdict was wrong +> +> The first revision concluded that ESI was **structurally blocked**: that it +> presupposed a TS-owned template cache which did not exist, and that such a cache was +> in turn blocked on purge capability the platform did not offer. **Both claims are +> false**, and an external review was right to reject them. +> +> Verified against the pinned `fastly` 0.12.1: +> +> - **The cache boundary is native.** `fastly::cache::core` provides +> `insert(key, max_age).execute() -> StreamingBody` for arbitrary bytes, `lookup()` / +> `found()` to read them back, and `Transaction` with `must_insert()` for request +> collapsing. The two-stage design needs no separate KV or template service. +> - **Purge exists in-process.** `InsertBuilder::surrogate_keys([...])` attaches keys at +> insert; `fastly::http::purge::purge_surrogate_key` purges from inside Compute. The +> management-API token scope cited in the original is irrelevant to it. +> - **The original pipeline ordering was backwards.** It said "order esi → lol*html, +> never the reverse." `lol_html` \_emits* the `esi:include` tags, so ESI must run after +> it. Correct order is in [§6.6](#66-the-esi-pipeline-corrected). +> +> The error was inspecting what this repository does and reporting it as what the +> platform permits — the same mistake this document criticises #1009 for making in the +> other direction. +> +> **ESI is therefore feasible and unvalidated, not rejected.** Validating it is +> [a separate plan](../plans/2026-08-10-1009-esi-validation-spike.md). What survives +> here is the latency re-diagnosis and the Stage 0 optimisation, which are worth doing +> and are **not** an answer to #1009. + +**Decision requested:** approve the four items in §1. > **Orientation.** #1009 asks whether Edge Side Includes (ESI) can cache page fragments > so that cacheable publisher HTML is separated from per-user ad state, recovering a > TTFB regression that Trusted Server (TS) adds to navigations on a Next.js App Router -> publisher running on Fastly Compute. The answer is no to ESI, and the regression has a -> cheaper cause than the issue assumes. +> publisher running on Fastly Compute. ESI can do this; whether it should is not settled +> here. Separately, the regression has a cheaper cause than the issue assumes. > > **This document deliberately carries no performance measurements.** Every conclusion > below is derived from code at the pinned baseline, so it can be checked by reading the @@ -19,22 +47,24 @@ about three days of measurement. > it is named as unknown and [§3](#3-monday-morning) says how to obtain it. > > Terms used throughout: **the hold** = TS holding the HTTP response open at `` -> until the server-side auction (SSAT) resolves. **#418** = a React hydration-mismatch -> defect caused by `adInit()` mutating ad-slot subtrees during hydration; it is why bid -> application is deferred to `window.load`. **The SSAT price defect** = a live -> mispricing bug named in #1009 (prices reading 100× high) — cited from #1009 and prior -> investigation, not re-verified here. +> until the server-side auction (SSAT) resolves. **React #418** = the React +> hydration-mismatch error raised when `adInit()` mutates ad-slot subtrees during +> hydration; it is why bid application is deferred to `window.load`. It is a React error +> number, **not** a repository issue — the tracker is +> [#938](https://github.com/IABTechLab/trusted-server/issues/938). **The SSAT price +> defect** = a live mispricing bug named in #1009 (prices reading 100× high) — cited +> from #1009 and prior investigation, not re-verified here. --- ## 1. Decision requested -| # | Decision | Owner needed | -| --- | -------------------------------------------------------------------------------------------------------------------------------- | ------------- | -| D1 | **ESI is deferred.** Revival condition: #418 resolved _and_ the `window.load` gate removed. Not a rejection — a dated condition. | Eng + product | -| D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | -| D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to the `Vary` check in §3. | Eng | -| D4 | **Stages 1–2 queue behind the SSAT price defect and #418.** Stages 3b–5 unscheduled. | Product | +| # | Decision | Owner needed | +| --- | --------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| D1 | **ESI is feasible and unvalidated.** Validate it via [the spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), not by deferring it. | Eng + product | +| D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | +| D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to the `Vary` check in §3. | Eng | +| D4 | **Stages 1–2 queue behind the SSAT price defect and #938.** Stages 3b–5 unscheduled. | Product | Rationale for D4 in [§8](#8-priority). Everything this document recommends _against_ doing is in [§7](#7-deferred-work-specified-not-scheduled), at deliberately lower @@ -44,17 +74,20 @@ detail than the work it recommends. ## 2. Why — the three findings -**ESI does not work here, for a structural reason #1009 misses.** ESI's input is pull -(`BufRead`); `lol_html`'s is push (`HtmlRewriter::write`). ESI cannot sit downstream of -the rewriter without an intermediate buffer, and in the two-stage design the cache -boundary _is_ that buffer. **ESI presupposes a TS-owned template cache** rather than -being independent of one — and that cache is blocked on purge capability TS does not -have (no `Surrogate-Key` anywhere; the Fastly management token is scoped without purge -permission). ESI is also Fastly-only at every API level. Its one advantage over a -client fetch — no round trip — is worth nothing while bids are not consumed until -`window.load`. Separately, enabling ESI's Dynamic Content Assembly would be an SSRF -vector: bid payloads carry partner-controlled creative markup, so an SSP could embed -`` and make the edge fetch an arbitrary URL. Details in +**ESI is buildable on the pinned SDK, and unvalidated.** `lol_html` emits +`esi:include` tags into a shared template; `fastly::cache::core` stores that template; +the `esi` crate assembles per request on the way out. Everything that requires is +already a dependency. The real open questions are empirical, not architectural: does it +beat a plain client fetch by enough to justify a Fastly-only rendering path, and can +per-user leakage be excluded under cold MISS, warm HIT, stale revalidation, and fragment +failure. [The spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) answers +those; [§6.6](#66-the-esi-pipeline-corrected) gives the pipeline. + +Two constraints stay true regardless. ESI is **Fastly-only at every API level**, so it +is a per-platform accelerator rather than the architecture, and its maintenance cost +belongs in the decision. And its Dynamic Content Assembly must be **explicitly disabled** +— bid payloads carry partner-controlled creative markup, so under `DcaMode::Esi` an SSP +could embed `` and make the edge fetch an arbitrary URL. Details in [Appendix E](#appendix-e--esi-notes-condensed). **The auction is already out of band; the hold is ~free.** It is dispatched _before_ @@ -361,7 +394,7 @@ no post-processor takes the streaming path and would see a lower floor. ### 6.5 Confidence **High on the structural claims.** §6.2's argument, the bypass forcing a cache miss, the -ESI push/pull mismatch, the silent-empty-bids failure mode, the geo and purge blockers, +the silent-empty-bids failure mode, the geo and `Vary` blockers, and the fill-canary blindness are all read directly out of the code at `cfb98f4`. Anyone can check them without running anything. @@ -373,6 +406,51 @@ That is a caution about small samples generally, not only about that one — whi §3 Step C specifies the measurement rather than this document supplying a substitute for it. +### 6.6 The ESI pipeline, corrected + +An earlier revision of this document said "order esi → lol*html, never the reverse." +That is backwards. `lol_html` is what \_emits* the `esi:include` tags; ESI cannot process +tags that do not exist yet. The correct order: + +``` +origin → lol_html transform → fastly::cache::core → esi assemble → finalize → client + (emit esi:include at the (shared template, (per request, (EC cookie, + head + body-close seams, surrogate-keyed, fetch the geo, privacy + no per-user data) TS-chosen TTL) bids fragment) net) +``` + +The push/pull mismatch that the earlier revision treated as a blocker is real but +irrelevant: `lol_html` pushes, `esi` pulls, and **the cache is the buffer between them**. +That is not an obstacle to the two-stage design — it _is_ the two-stage design, which is +what #1009 proposed in the first place. + +Mechanism, all present in the pinned `fastly` 0.12.1: + +| Need | API | +| ----------------------- | ----------------------------------------------------------------------------------- | +| Store the template | `cache::core::insert(key, max_age).execute() -> StreamingBody` | +| Read it back | `cache::core::lookup(key)` → `found()` | +| Avoid a thundering herd | `cache::core::Transaction` — `must_insert()` / `must_insert_or_update()` | +| Invalidate | `InsertBuilder::surrogate_keys([...])` + `fastly::http::purge::purge_surrogate_key` | + +Purge runs **inside Compute**. The management-API token scope cited under +[Stage 4](#7-deferred-work-specified-not-scheduled) governs a different surface and does +not gate this. + +**Three caches, kept distinct.** Conflating them is what produced the original error: + +1. **Origin read-through** — raw origin bytes. What Stage 0 turns back on. +2. **Shared transformed template** — post-`lol_html`, pre-ESI, no per-user data. The ESI + target, and new. +3. **Assembled-response delivery cache** — the final per-user output. **Must never + exist.** Nothing in this document or the spike proposes one. + +**Validation constraint.** Viceroy 0.17 cannot exercise the customized read-through hooks +end to end. Unit tests can cover the transform and the security properties; MISS / HIT / +stale / shielding behaviour must run against a real Fastly test service. + +--- + --- ## 7. Deferred work, specified not scheduled @@ -425,12 +503,19 @@ and can never be shared-cached ([Appendix C](#appendix-c--vary-signals-condensed **Also gated on Step B**: if nothing consumes TS's response headers, this tier is inert until a topology change. -**Stage 4 — purge capability.** Not sized. Prerequisite for anything beyond the backend -readthrough cache. TS today has no `Surrogate-Key` emission and no purge permission, so +**Stage 4 — purge wiring.** Not sized. Prerequisite for a TS-owned cache. TS today emits +no `Surrogate-Key` and holds a management token scoped without purge — but that token is +the wrong surface: `InsertBuilder::surrogate_keys` and +`fastly::http::purge::purge_surrogate_key` are both in the pinned SDK and purge runs +inside Compute ([§6.6](#66-the-esi-pipeline-corrected)). This is **missing wiring, not a +platform limit.** Until it exists, any TS-owned cache is TTL-only and a config push takes up to one TTL to take effect. -**Stage 5 — TS-owned template cache, then ESI.** Not sized, and gated on D1's revival -condition. +**Stage 5 — ESI.** Superseded. ESI no longer waits on a "revival condition"; it is +feasible on the pinned SDK and is validated by +[its own spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), which does not +queue behind Stages 1–4. The shared template cache it needs is +`fastly::cache::core` ([§6.6](#66-the-esi-pipeline-corrected)), not a new service. **Identity needs no work.** A new visitor's first navigation sets the EC cookie and the privacy net downgrades that one response; every later navigation sets no cookie and is @@ -443,10 +528,10 @@ server-side (`rsc_flight.rs` plus `integrations/nextjs/`, ~4,100 lines) by rewri hydration's _input_; a shim would be a second source of truth producing the exact mismatch both exist to prevent. Late-bid rendering has no foothold and the obvious interception point is measured-unsafe: a controlled capture found the `__next_f` gate -reproducing #418 on every run and destroying the creative on half of them — it patches +reproducing React #418 on every run and destroying the creative on half of them — it patches `__next_f.push` shortly after React's first commit, while hydration continues for thousands more. Two retractions to carry forward: that gate is -measured-unsafe rather than merely unproven, and "#418 at ~5% and not impression-costing" +measured-unsafe rather than merely unproven, and "React #418 at ~5% and not impression-costing" is retracted — it came from pages whose slots are not React-owned. Note `docs/superpowers/specs/2026-07-24-adinit-hydration-gate-design.md` exists only on the unmerged branch `958-adinit-hydration-chunk-gate`, so `publisher.rs:3461-3464` points at @@ -472,10 +557,11 @@ it did not need to opt out of. **Stages 1–2 queue behind the correctness defects.** Their failure mode is silent revenue loss, against a publisher whose ads currently fill reliably. The SSAT price defect misprices live auctions, and **a slow correct auction loses less money than a -fast wrong one**. #418 sits ahead too: Stage 1's join gate must be reconciled with +fast wrong one**. React #418 sits ahead too: Stage 1's join gate must be reconciled with whatever hydration gate lands, and doing that twice is waste. -**Stages 3b–5 are not competitive** on current evidence and should not be scheduled. +**Stages 3b–4 are not competitive** on current evidence and should not be scheduled. +**ESI is no longer in this queue** — it is validated separately and on its own evidence. --- @@ -518,7 +604,8 @@ Rows 1–6 are in [§6.1](#61-corrections-to-1009s-premises). The remainder: **Cache tiers.** T1 backend readthrough (already available; TS opts out). T2 TS-owned template cache (adds KV latency, eventual consistency, a full invalidation design). T3 delivery cache of TS output (Fastly topology change; **ESI cannot run**, since -Compute is not invoked on a HIT). T2 and T3 both introduce a cache TS cannot purge. +Compute is not invoked on a HIT). T2 and T3 both introduce a cache TS does not yet purge — +wiring that is available but unbuilt. --- @@ -607,22 +694,25 @@ exists. ## Appendix E — ESI notes (condensed) -For if and when [D1](#1-decision-requested)'s revival condition is met. Expand then; -recording only what would otherwise be re-derived: +Input to [the ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md). +Recording only what would otherwise be re-derived: - Pin `esi = "0.7"`. Pre-1.0, irregular cadence, two yanked betas in the 0.7 line. - **Use `process_stream`, not the wrappers.** `process_response` and `process_response_streaming` consume `self` _and_ send the response themselves, taking ownership away from the finalize / `ec_finalize` ordering. -- **Order esi → lol_html**, never the reverse, via a newtype implementing `io::Write`. - Mind the `StreamingBody`-is-a-`BufWriter` hazard: esi flushes per parse batch, so any - adapter in between must propagate `flush()`. +- **Order lol_html → cache → esi.** `lol_html` emits the tags; ESI consumes them on the + way out. An earlier revision had this backwards — see + [§6.6](#66-the-esi-pipeline-corrected). Mind the `StreamingBody`-is-a-`BufWriter` + hazard on the way to the client: esi flushes per parse batch, so anything between esi + and the `StreamingBody` must propagate `flush()`. - **Always supply a custom fragment dispatcher.** The built-in one builds a dynamic backend per URL host and panics on a hostless URL; dynamic backends are also the known Viceroy local-dev failure mode here. -- **DCA off, asserted explicitly** — not merely left at its default. Rationale is the SSRF - vector in [§2](#2-why--the-three-findings): partner-controlled creative markup would - become ESI-executable at the edge. +- **DCA off, asserted explicitly** — not merely left at its default. Rationale is the + SSRF vector in [§2](#2-why--the-three-findings): partner-controlled creative markup + would become ESI-executable at the edge. Pair it with an **exact-path allowlist + dispatcher**, so a fragment URL that is not the bids endpoint cannot be fetched at all. - **`` runs _all_ attempts and concatenates every non-failed output** — not first-success-wins, so primary/fallback pairs render both. Least obvious behaviour in the crate. @@ -649,34 +739,35 @@ queued before it loads (#1009 Part 1) — not filed, should be. All pinned to `cfb98f4`. -| Concern | Location | -| ------------------------------------------- | ------------------------------------------------------------------------------- | -| Eligibility decision | `publisher.rs:2651`, `:2660` | -| `is_navigation_request` | `http_util.rs:73-98` | -| Auction dispatch (pre-origin, non-blocking) | `publisher.rs:2698-2760` | -| Auction overlap intent | `auction/orchestrator.rs:950-952` | -| Auction timeout resolution | `publisher.rs:2680-2684` (creative_opportunities, else auction.timeout_ms) | -| Conditional/range header strip | `publisher.rs:2832-2836` | -| Origin cache bypass | `publisher.rs:2866-2868` | -| Origin 304 → 502 guard | `publisher.rs:2894-2916` | -| `adSlots` build (per-URL) | `publisher.rs:2920`, `:3558-3577`, `:3501-3525` | -| Uncacheable stamp | `publisher.rs:2945-2963` | -| `` hold — sync / async / Fastly lazy | `publisher.rs:2235` / `:2109` / `:1318-1390` | -| Hold buffer | `publisher.rs:2177-2218` | -| Auction collect (HTML / non-HTML) | `publisher.rs:2431` (emits `:2456`) / `:2388` (emits `:2410`) | -| Abandonment emitter | `publisher.rs:2360` | -| Bids script build | `publisher.rs:3438-3491` | -| `/_ts/page-bids` | `publisher.rs:3611`, handler `:3723`, auction `:3903` | -| Injection seams (head / body-close) | `html_processor.rs:310-363` / `:381-395` | -| Post-processor buffering | `html_processor.rs:62-94` | -| Next.js post-processor registration | `integrations/nextjs/mod.rs:107` | -| Max buffered body (16 MB) | `settings.rs:77-79` | -| EC cookie issuance policy | `ec/finalize.rs:86-107` | -| Cookie-privacy net | `response_privacy.rs:20-61` | -| Geo response headers | `adapter-fastly/src/middleware.rs:194-200` | -| Cacheable-header precedent | `http_util.rs:294-311` | -| No purge permission | `adapter-fastly/src/management_api.rs:12` | -| Client initial-ad gate | `js/lib/src/integrations/gpt/index.ts:536-555` | -| `adInit` bid application | `js/lib/src/integrations/gpt/index.ts:566`, `:652`, `:657-661` | -| Client SPA auction hook | `js/lib/src/integrations/gpt/index.ts:806`, `:859`, `:892-949` | -| Prior design that introduced the killers | `docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md` | +| Concern | Location | +| -------------------------------------------- | ------------------------------------------------------------------------------- | +| Eligibility decision | `publisher.rs:2651`, `:2660` | +| `is_navigation_request` | `http_util.rs:73-98` | +| Auction dispatch (pre-origin, non-blocking) | `publisher.rs:2698-2760` | +| Auction overlap intent | `auction/orchestrator.rs:950-952` | +| Auction timeout resolution | `publisher.rs:2680-2684` (creative_opportunities, else auction.timeout_ms) | +| Conditional/range header strip | `publisher.rs:2832-2836` | +| Origin cache bypass | `publisher.rs:2866-2868` | +| Origin 304 → 502 guard | `publisher.rs:2894-2916` | +| `adSlots` build (per-URL) | `publisher.rs:2920`, `:3558-3577`, `:3501-3525` | +| Uncacheable stamp | `publisher.rs:2945-2963` | +| `` hold — sync / async / Fastly lazy | `publisher.rs:2235` / `:2109` / `:1318-1390` | +| Hold buffer | `publisher.rs:2177-2218` | +| Auction collect (HTML / non-HTML) | `publisher.rs:2431` (emits `:2456`) / `:2388` (emits `:2410`) | +| Abandonment emitter | `publisher.rs:2360` | +| Bids script build | `publisher.rs:3438-3491` | +| `/_ts/page-bids` | `publisher.rs:3611`, handler `:3723`, auction `:3903` | +| Injection seams (head / body-close) | `html_processor.rs:310-363` / `:381-395` | +| Post-processor buffering | `html_processor.rs:62-94` | +| Next.js post-processor registration | `integrations/nextjs/mod.rs:107` | +| Max buffered body (16 MB) | `settings.rs:77-79` | +| EC cookie issuance policy | `ec/finalize.rs:86-107` | +| Cookie-privacy net | `response_privacy.rs:20-61` | +| Geo response headers | `adapter-fastly/src/middleware.rs:194-200` | +| Cacheable-header precedent | `http_util.rs:294-311` | +| Management token lacks purge (wrong surface) | `adapter-fastly/src/management_api.rs:12` | +| In-process purge / surrogate keys | `fastly` 0.12.1 `cache::core`, `http::purge::purge_surrogate_key` | +| Client initial-ad gate | `js/lib/src/integrations/gpt/index.ts:536-555` | +| `adInit` bid application | `js/lib/src/integrations/gpt/index.ts:566`, `:652`, `:657-661` | +| Client SPA auction hook | `js/lib/src/integrations/gpt/index.ts:806`, `:859`, `:892-949` | +| Prior design that introduced the killers | `docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md` | From bc6b4001effa9fb4d26e1c4c1d62531a61aab243 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 15:35:18 +0530 Subject: [PATCH 170/395] Fix the docs build and the design breaks in the #1009 spike MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docs build was broken and committed. `npm run build` failed with 27 dead links from the spec's relative `../../../crates/...` references; VitePress rejects links outside the docs root and no other spec in the repository uses them. I had only ever run `npm run format`, which does not catch this. Converted to plain code references, matching what every other spec does. Build passes. Four design breaks, all verified against the source before fixing. The shared template was not request-neutral. `tsjs.adSlots` was kept in it on the grounds of being per-URL. Its content is per-URL; its presence is gated on should_run_ad_stack, which depends on consent, bot classification, prefetch status and the auction kill switch. The first request to fill the cache would have frozen its own consent decision into an object every later visitor reads. Both slots and bids now move to the request-aware fragment, the template carries an unconditional inert placeholder, and a test asserts the template is byte-identical across requests differing in consent, bot and prefetch state. The Core Cache pseudocode did not compile. surrogate_keys takes and returns self, so the sample discarded the builder and then used a moved binding; execute() yields a write stream rather than the readable object the next step assumed; finish() was never called; and the key omitted the assembly mode, so the client-fill and ESI arms would have poisoned each other. Replaced with a transaction using execute_and_stream_back, an explicit user_metadata envelope since cache::core carries no HTTP semantics, a cancel-on-error path, and a versioned key. The alternative read-through design is named rather than assumed. The ESI fragment contract was broken. It pointed at /_ts/page-bids, which returns JSON, and ESI splices fragment bytes literally — the page would have contained raw JSON where an executable script belongs. Also: the endpoint's same-origin gate rejects internal subrequests, parent identity and consent context did not propagate, root dispatch was not suppressed so spend would double, and path-only validation admits an attacker authority. The no-C3 gate only forbade public, s-maxage and Surrogate-Control. A bare max-age=60 passes that and is still shared-cacheable — and is exactly what the measured origin sends. Now requires private, no-store positively, tested for returning users, who set no EC cookie and so are not covered by the privacy net. Stage 0 could still ship on provisional evidence: the findings said PROVISIONAL PASS but the plan said Step A had passed and the gate accepted only PASS or FAIL. There are now three verdicts, with FINAL PASS requiring a real session cookie, Basic Auth through TS, the experiment variant, representative routes and cached-hit render attribution. Methodology: A3 and A2 are no longer compared on root TTFB, since both serve the same template — the comparison is bids-ready, adInit fire and first attributed creative paint. Sample plan gains allocation, randomization, pilot variance, MDE and power, CI method and carryover control. Correlation becomes a lineage ID carrying the experiment arm through fragment and auction telemetry, since a root-only ID never reaches an auction that runs in a subrequest. C1 and C2 cache status are recorded separately. DCA now calls the setters rather than commenting that defaults suffice, and fragment caching is disabled. Corrected: Viceroy 0.17 does support cache::core locally; only the customized HTTP read-through hooks are unsupported. Also removed leftovers claiming KV latency for what is a cache, and a config-only rollback. --- ...2026-08-08-1009-measurement-and-stage-0.md | 33 +- .../2026-08-10-1009-esi-validation-spike.md | 289 ++++++++++++++---- ...08-esi-cacheable-root-validation-design.md | 75 ++--- 3 files changed, 293 insertions(+), 104 deletions(-) diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md index 3aa100ade..6298ca2c6 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md @@ -197,9 +197,9 @@ diff <(norm nc_a.html) <(norm ck.html) | head -20 Send the `Host` override — the origin is a shared vhost and will not return the right document without it. Read it from `publisher.origin_host_header_override`. -**Step A has already been run and passed.** See -[the findings](./2026-08-08-1009-measurement-findings.md). Re-run only if the origin -changes. +**Step A has been run once and returned a PROVISIONAL PASS**, which is **not** sufficient +to flip the flag. See [the findings](./2026-08-08-1009-measurement-findings.md) for the +five untested conditions. Complete them and record a `FINAL PASS` before Task 5. Three failure shapes, any of which blocks Stage 0 independently of the `Vary` verdict: @@ -750,12 +750,29 @@ git commit -m "Record server-side latency breakdown for #1009" ## Task 5: Stage 0 — turn the origin cache bypass off -**Gate:** do not flip the flag until Task 1 has a recorded verdict. +**Gate:** do not flip the flag until Task 1 has recorded a **`FINAL PASS`**. There are +three verdicts, not two. -- **PASS** → Task 5a (config flip). -- **FAIL** → Task 5b. Do **not** flip on a FAIL; it can serve an RSC payload to an HTML +- **`FINAL PASS`** → Task 5a (config flip). +- **`PROVISIONAL PASS`** → **stop.** Not a release gate. This is the current state. It + means the representation split is declared correctly under the conditions tested, and + that those conditions were too narrow to flip production on. +- **`FAIL`** → Task 5b. Do **not** flip; it can serve an RSC payload to an HTML navigation. +**`FINAL PASS` requires all five, each recorded in the findings document:** + +| Condition | Why the provisional run is insufficient | +| -------------------------------------------------------- | ---------------------------------------------------- | +| A real authenticated or state-bearing session cookie | `sessionid=abc123` is synthetic and proves nothing | +| Basic Auth exercised **through TS**, not just the origin | #1009 describes a gated deployment | +| The experiment variant named in #1009 | Absent from the origin's `Vary`; unexplained | +| Representative routes — article, section, search | Only the homepage was probed | +| Cached-hit slot and render attribution | The randomized div IDs are an unverified interaction | + +Any one of these unrecorded means the verdict stays `PROVISIONAL PASS` and Task 5 does +not start. + Because Task 3 made the bypass config-driven, Stage 0 ships as a **config change on an already-deployed build**. No second release, and rollback is another config push rather than a revert. That matters here specifically: the failure mode this gates on is cache @@ -982,8 +999,12 @@ Named so nobody widens this plan mid-flight. All are specified in the spec. N where applicable, and the consequence spelled out. - [ ] `publisher_timing` and `publisher_hold` lines are emitted in production and readable, and `hold_wait_ms` has a recorded median. +- [ ] Task 1 recorded a **`FINAL PASS`** — all five conditions in Task 5's gate closed, + not merely the provisional run. - [ ] Either Task 5a is shipped, or Task 1 returned FAIL and both a production defect and a follow-up plan for 5b exist. +- [ ] A purge path or versioned cache-key namespace exists **before** the flip, or the + "wait out the TTL" rollback is explicitly accepted and recorded as a risk. - [ ] **The win is measured client-side, not from `origin_fetch_ms`.** That figure is origin TTFB and excludes body download, rewrite, and post-processing — it is attribution, not the outcome. #1009 already has a working tester-cookie browser A/B diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index bbfd7de84..770cb05cc 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -76,6 +76,12 @@ A0→A1 measures the bypass. A1→A2 measures the template split. A2→A3 measur client assembly — **that difference is the entire case for ESI**, and it is the number this plan exists to produce. +**Do not compare A2 and A3 on root TTFB.** They serve the same C2 template, so their root +timings should be near-identical by construction; a null result there proves nothing. +ESI's claimed advantage is that bids arrive without a client round-trip, so measure: +**bids-ready time**, **`adInit` fire time**, and **first TS-attributed creative paint**. +Root TTFB stays as a guard that the template path did not regress, not as the comparison. + REF is included because #1009 anchors on it, and excluded from pass/fail because TS-off does no auction and no injection. Comparing against it measures the feature's existence, not its implementation. @@ -150,9 +156,15 @@ git commit -m "Add the esi crate to the Fastly adapter for the #1009 validation ## Task 2: Stand up the test service and the harness -**Viceroy 0.17 cannot exercise the `cache::core` hooks end to end.** Unit tests cover the -transform and the security properties; MISS / HIT / stale / shielding must run on a real -Fastly service. Establish that before building, or Tasks 3–6 have nowhere to run. +**Viceroy 0.17 does support `cache::core` locally** — an earlier draft of this plan said +otherwise and was wrong. What it does **not** support is the customized HTTP +read-through hooks (`after_send` / `set_body_transform`), which matters only if the +alternative design in Task 3 Step 4 is chosen. + +So: C2 insert/lookup/transaction logic, the transform, and the security properties are all +testable locally. **Shielding, request collapsing under real concurrency, POP behaviour, +and stale revalidation are not** — those need a real Fastly service. Establish one before +Tasks 3–6, and be clear which findings came from which environment. - [ ] **Step 1: Provision a dedicated test service** @@ -169,30 +181,56 @@ The shielding answer also settles an open question from the Stage 0 findings: #1 off-TS win came from a shield HIT, so whether the test service has one determines whether its numbers transfer to production at all. -- [ ] **Step 2: Extend the harness for correlation** +- [ ] **Step 2: Extend the harness for lineage, not just correlation** + +The existing tester-cookie A/B has no way to join server timings to browser timings. A +root-only request ID is not enough either: under A3 the auction happens in a **fragment +subrequest**, so a root ID never reaches the auction telemetry. -The existing tester-cookie A/B has no way to join server timings to browser timings. Add -a per-request correlation ID — generated at TS entry, echoed in an `x-ts-request-id` -response header, and included in every timing log line. +Propagate a **lineage ID plus the experiment arm** through the whole chain: -Without it, the experiment cannot join hold time, origin time, auction telemetry, browser -TTFB, and render outcome for the same request. **That is the difference between an +``` +root request → C2 lookup → fragment subrequest → auction telemetry → browser render event +``` + +Generated at TS entry, forwarded into the fragment request, attached to the +`auction_events_raw` row, echoed as `x-ts-request-id`, and exposed to the browser harness +so render events carry it. Every timing log line includes both fields. + +Without this the experiment cannot join hold time, origin time, auction telemetry, browser +TTFB, and render outcome for the same pageview. **That is the difference between an experiment and a pile of numbers.** -- [ ] **Step 3: Capture cache tier and status per request** +- [ ] **Step 3: Capture C1 and C2 status separately** -Record `x-cache`, `hit-state`, `age`, and the serving POP alongside each measurement. A -median that mixes cold-MISS and warm-HIT requests is meaningless, and arms cannot be -compared unless the mix is known. +`x-cache`, `hit-state`, and `age` describe the **HTTP read-through cache (C1)**. They say +nothing about the **transformed-template cache (C2)**, which is a `cache::core` object +with no HTTP semantics. Recording only the former and calling it "cache status" would +attribute C2 hits and misses to the wrong tier. + +Emit both: the C1 headers as-is, plus an explicit `x-ts-c2` field carrying HIT / MISS / +STALE / BYPASS from the transaction outcome. Record the serving POP alongside. A median +that mixes cold-MISS and warm-HIT requests is meaningless, and arms cannot be compared +unless the mix is known — per tier. - [ ] **Step 4: Define the sample plan before collecting anything** -State, in the findings document, ahead of time: requests per arm per route, how cold MISS -is forced, how warm HIT is confirmed, and the confidence interval to be reported. +Write all of this into the findings document **before** the first measurement, and treat +it as fixed: + +| Element | What to state | +| -------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Allocation | Requests per arm per route, and how arms are assigned | +| Randomization | Randomized or blocked by route and cache state — not sequential runs | +| Pilot variance | A small pilot to estimate variance, before sizing the real run | +| MDE and power | The smallest difference worth detecting, and the N that detects it | +| CI method | Which interval, computed how | +| Warmup and carryover | How cold MISS is forced, how warm HIT is confirmed, and how one arm's cache state is prevented from contaminating the next | Rationale: this whole effort exists because #1009 drew a causal conclusion from N=4 that -did not survive contact with the code. Repeating that with more arms would be worse, not -better. +did not survive contact with the code. Repeating that with more arms and no power +calculation would be worse, not better — it would look rigorous while being equally +unfalsifiable. --- @@ -229,51 +267,127 @@ already documents: `Settings` carries `#[serde(deny_unknown_fields)]`, `ts confi typed, and `Publisher` has a hand-written `Default` plus eight exhaustive test literals and a live doctest. -- [ ] **Step 2: Emit markers instead of inlining, under `ClientFill`/`Esi`** +- [ ] **Step 2: Make the template strictly request-neutral** + +**The obvious design is wrong and would leak.** An earlier draft kept `tsjs.adSlots` in +the shared template on the grounds that it is per-URL. Its _content_ is per-URL; its +_presence_ is not. It is gated on `should_run_ad_stack` (`publisher.rs:2920-2927`), which +is `is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the first request to fill C2 would freeze **its own** consent decision, bot +classification, prefetch status, and kill-switch state into an object every later visitor +reads. A consent-denied first fill serves a no-ads template to consenting users; a +consenting first fill serves ad markup to a user who refused. + +**Rule: the template contains an unconditional inert placeholder and nothing else.** + +| Element | Where it lives | +| ------------------------- | -------------------------------------------------- | +| tsjs bundle script tag | Template — content-hashed, genuinely per-URL | +| URL rewrites | Template — per-host, in the cache key | +| `tsjs.adSlots` | **Fragment** — its presence is request-dependent | +| `tsjs.bids` | **Fragment** | +| GPT diagnostics bootstrap | **Fragment** — gated on a per-request cookie/query | + +Emit **one** unconditional marker at the body-close seam, identical on every request that +reaches the transform. Under `Esi` it is an ``; under `ClientFill` it is +nothing at all, with the client fetching unprompted. + +- [ ] **Step 3: Bypass C2 for anything that must not be shared** + +`cache::core` is not an HTTP cache — it will happily store whatever you hand it. Nothing +rejects private or authenticated responses for you. Refuse to insert when **any** holds: + +- The origin response carries `Set-Cookie`. +- The origin response is `private`, `no-store`, or `no-cache`. +- The request carried `Authorization`. +- The response is not 200 with an HTML content type. +- DataDome's request filter replaced the document. -The two seams are already isolated — that is #1009's correct observation. At head-open, -`tsjs.adSlots` is **per-URL and stays in the template** (config- and path-derived only, -`publisher.rs:3501-3525`). At body-close, emit a marker instead of the bids script. +Audit every request-dependent rewrite before declaring the template neutral — the +integration head-inserts and the GPT-diagnostics bootstrap are both request-scoped and +must not reach C2. -Under `Esi`: ``. -Under `ClientFill`: nothing at all — **not an empty bids script.** The Stage 0 plan -explains why: an empty script calls `scheduleInitialAdInit({})` and assigns -`ts.bids = {}` synchronously, racing the client fetch. +**Assert it, do not assume it.** A unit test over the transform output must fail on any +of: a bid value, an EC ID, a consent string, a geo value, a diagnostics bootstrap, or a +`Set-Cookie`. Then a second test must assert the template is **byte-identical** for two +requests differing in consent, bot classification, and prefetch status. That second test +is the one that catches this class of bug; the first would have passed on the broken +design. -**Assert the template carries no per-user bytes.** A unit test over the transform output -must fail on any of: a bid value, an EC ID, a consent string, a geo value, or a -`Set-Cookie`. This is the test that makes C2 safe, and it is cheaper to write now than to -retrofit. +- [ ] **Step 4: Write and read C2 — with the real API** -- [ ] **Step 3: Write the template into C2** +The builder is move-based and the insert and read handles are different objects. Naïve +code does not compile: ```rust -// Fastly adapter. Key on the same signals the origin varies on, plus TS's own -// variant inputs. Surrogate-key it so rollback can purge rather than wait. -let mut insert = fastly::cache::core::insert(cache_key, template_ttl); -insert.surrogate_keys([&surrogate_key_for_url, "ts-template"]); -let mut body = insert.execute()?; -// stream the lol_html output into `body` +// WRONG — surrogate_keys consumes the builder and returns it; this discards the +// return value and then uses a moved binding. And execute() gives a WRITE stream, +// so there is nothing to read back from it. +let mut insert = cache::core::insert(key, ttl); +insert.surrogate_keys(["ts-template"]); +let body = insert.execute()?; ``` -Use `cache::core::Transaction` with `must_insert()` for the lookup, so a cold cache under -load transforms once rather than per concurrent request. +Correct shape, using a transaction so a cold cache under load transforms once: -**Cache key must include** everything the origin's `Vary` names — `rsc`, +```rust +use fastly::cache::core::{Transaction, CacheKey}; + +let tx = Transaction::lookup(CacheKey::from(key_bytes)).execute()?; + +let template: Body = if let Some(found) = tx.found() { + found.to_body() // C2 HIT — skip origin fetch and transform +} else if tx.must_insert_or_update() { + // C2 MISS. Fetch origin, transform, insert, and read our own bytes back in one + // pass: execute_and_stream_back gives both the write handle and a readable Found. + let (mut writer, found) = tx + .insert(template_ttl) + .surrogate_keys(["ts-template", &url_surrogate_key]) // chained, not discarded + .user_metadata(metadata_envelope) // see below + .execute_and_stream_back()?; + stream_lol_html_output_into(&mut writer)?; + writer.finish()?; // REQUIRED + found.to_body() +} else { + unreachable!("transaction must either find or be obliged to insert") +}; +``` + +`finish()` is not optional — without it the object never completes and its length stays +unknown. On any transform error, **cancel rather than finish**, or a partial template is +inserted and served to everyone until it expires. + +**`cache::core` carries no HTTP semantics.** Status, headers, content encoding, and +revalidation are all yours. Serialize what you need into `user_metadata` — at minimum the +content encoding, the transform schema version, and the origin `Vary` values the key was +built from — and decide explicitly whether the stored template is compressed. + +**Cache key must include**, beyond the origin's declared `Vary` (`rsc`, `next-router-state-tree`, `next-router-prefetch`, `next-router-segment-prefetch`, -`Accept-Encoding` (measured, see the Stage 0 findings) — **plus** TS's own per-variant -inputs: request host and scheme, the enabled-integration set, and the tsjs content hash. -Per-user signals must never appear in the key; they must be absent from the template -instead. If a signal cannot be excluded from the template, it does not belong in C2. +`Accept-Encoding` — measured, see the Stage 0 findings): -Set `template_ttl` deliberately short for the spike. A short TTL bounds every failure mode -here and costs only hit rate. +- The full URL, explicitly. Do not rely on an ambient request key. +- **The assembly mode.** A2 and A3 emit different template bytes and would otherwise + poison each other's entries. +- **A template schema version**, bumped whenever the transform changes, so a deploy does + not read yesterday's shape. +- Request host and scheme, the enabled-integration set, and the tsjs content hash. -- [ ] **Step 4: Read it back and assemble** +Per-user signals must never appear in the key. If a signal cannot be excluded from the +template, it does not belong in C2 at all. -On `found()`, skip the origin fetch and the transform entirely; hand the cached body to -the assembler. On miss, transform and insert as above, then assemble from what was -inserted. +**Design choice to make explicitly before writing code.** Two viable shapes: + +1. **Read-through with `after_send` + `set_body_transform`** — keeps HTTP semantics, + revalidation, and stale handling for free; less control over the key. +2. **`cache::core` as above** — full control; you own metadata, revalidation, and the + stale state machine. + +This plan assumes (2). If (1) is chosen, Step 4 is rewritten and the metadata envelope +disappears. Either way, the platform boundary must sit **before** the origin request, or +a C2 HIT cannot actually skip the fetch — which is the entire point. - [ ] **Step 5: Unit tests, then the target suite** @@ -328,12 +442,19 @@ it is the difference between a correct response and a leaked one. ```rust let config = esi::Configuration::default() - .with_escaped(false); -// default_dca and inherit_parent_dca stay at DcaMode::None / false — set them -// explicitly rather than relying on defaults; this is a pre-1.0 crate and the -// setting fails open. + .with_escaped(false) + .with_default_dca(esi::DcaMode::None) // call the setter; do not rely on the default + .with_inherit_parent_dca(false); ``` +Comments are not configuration. An earlier draft said DCA "stays at its default" — on a +pre-1.0 crate whose default could move in a patch release, and where this setting fails +**open**, that is not good enough. Call the setters. + +Also disable **fragment caching** explicitly, or mark the include `no-store="on"`. A +cached auction fragment is a per-user object in a shared cache — the C3 failure mode by +another route. + The dispatcher must be **exact-path allowlisted**: a fragment URL that is not the bids endpoint is refused, not fetched. The built-in dispatcher builds a dynamic backend per URL host and panics on a hostless URL — never use it. @@ -343,20 +464,57 @@ recursive parse would let an SSP make the edge fetch an arbitrary URL. **Add a u that feeds `` through a creative payload and asserts no fetch is attempted.** -- [ ] **Step 3: Deterministic synthetic fragment first** +- [ ] **Step 3: The fragment must be a script, not the JSON endpoint** + +**`/_ts/page-bids` cannot be the ESI target.** It returns +`serde_json::json!({"slots":…, "bids":…})` (`publisher.rs:3987`), and ESI splices fragment +bytes in literally — the page would contain raw JSON where an executable script belongs. +Nothing would call `scheduleInitialAdInit`. + +Add a **dedicated fragment endpoint** returning the executable script — the same shape +`build_bids_script` produces today, plus the `adSlots` assignment that moved out of the +template in Task 3 Step 2. Either that, or use the `esi` crate's fragment-response +processor to wrap the JSON; the dedicated endpoint is simpler and easier to assert on. + +Three more things the naïve marker gets wrong: + +- **The same-origin gate will reject it.** `page_bids_request_allowed` + (`publisher.rs:3644`) requires `Sec-Fetch-Site: same-origin` or the `X-TSJS-Page-Bids` + header. An internal ESI subrequest carries neither. Give the fragment endpoint an + internal contract and a fixed backend rather than weakening that gate — it exists to + stop third parties burning SSP quota. +- **Parent context does not propagate.** EC identity, consent state, client IP, geo, User + Agent, and the correlation ID all live on the parent request. Forward an **explicitly + approved allowlist** of them into the fragment request. Forwarding everything is how a + fragment ends up more privileged than the parent. +- **Root dispatch must be suppressed.** The navigation path already dispatches an + auction. If A3 does not suppress it, every pageview runs two — doubling SSP and APS + spend. This applies to **A2 and A3 alike**. + +- [ ] **Step 4: Validate the whole URL, not the path** + +An exact-path allowlist alone permits `https://attacker.example/_ts/page-bids`. Validate +**scheme, authority, method, path, and query** — or better, ignore the marker's URL +entirely and dispatch to a fixed internal backend, treating the `esi:include` as a signal +rather than an address. + +Add a test that feeds `` +through a creative payload and asserts no outbound fetch is attempted. + +- [ ] **Step 5: Deterministic synthetic fragment first** -Before the real auction, point the include at a fixed-content endpoint. This separates -"does the pipeline assemble correctly" from "does the auction behave," and the two fail -very differently. Only once assembly is proven does the include move to -`/_ts/page-bids`. +Before wiring the real auction, point the include at a fixed-content endpoint. This +separates "does the pipeline assemble correctly" from "does the auction behave," and the +two fail very differently. Only once assembly is proven does the fragment become the real +one. -- [ ] **Step 4: Handle the flush hazard** +- [ ] **Step 6: Handle the flush hazard** `esi` flushes its output writer after each parse batch. Fastly's `StreamingBody` is a `BufWriter`, so anything between esi and it must propagate `flush()` or nothing leaves the Wasm heap. -- [ ] **Step 5: Fragment failure must degrade, not break** +- [ ] **Step 7: Fragment failure must degrade, not break** Assert that a fragment timeout or non-2xx yields a page with empty bids rather than a 5xx or a truncated document. Note the crate's non-obvious semantics: `alt` is attempted before @@ -385,9 +543,14 @@ Not a phase. Every one of these is a hard fail, independent of any performance r renders attributed. Use TS-attributed renders — the SSAT line item, non-empty `ts.bids`, `hb_adid` presence — **never slot fill**, which is blind to empty bids because `adInit` defines slots regardless. -- [ ] **No C3.** Assert the final assembled response is never shared-cacheable: no - `public`, no `s-maxage`, no `Surrogate-Control` on a response carrying per-user - state. +- [ ] **No C3 — assert positively, not by absence.** Forbidding `public`, `s-maxage`, and + `Surrogate-Control` is **not sufficient**: a bare `Cache-Control: max-age=60` passes + that check and is still shared-cacheable, and that is exactly what the measured + origin sends. Require instead that every assembled response carries + `Cache-Control: private, no-store` and that `Expires`, `ETag`, `Last-Modified`, and + all four CDN cache directives are stripped. Test it for **returning** users + specifically — they set no EC cookie, so the cookie privacy net never fires and is + not a backstop here. --- diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md index 1417c5122..a3a35002a 100644 --- a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -91,9 +91,9 @@ could embed `` and make the edge fetch an arbitrary URL. [Appendix E](#appendix-e--esi-notes-condensed). **The auction is already out of band; the hold is ~free.** It is dispatched _before_ -the origin fetch and does not block — dispatched at [publisher.rs:2751-2755](../../../crates/trusted-server-core/src/publisher.rs#L2751-L2755), sent at [:2870](../../../crates/trusted-server-core/src/publisher.rs#L2870) — +the origin fetch and does not block — dispatched at `publisher.rs:2751-2755`, sent at `:2870` — with a 500 ms budget. The actual cost is `with_cache_bypass` -([publisher.rs:2867](../../../crates/trusted-server-core/src/publisher.rs#L2867)), +(`publisher.rs:2867`), which forces every ad-eligible navigation to miss the Fastly readthrough cache. **The two fixes are multiplicative.** Removing the bypass alone lets the previously @@ -122,14 +122,14 @@ cheapest thing that unblocks anything. **Step B — what consumes TS's own response headers (under a day).** Request a TS-served path that already emits `public, s-maxage` -([http_util.rs:294-311](../../../crates/trusted-server-core/src/http_util.rs#L294-L311)) +(`http_util.rs:294-311`) twice and look for `x-cache`/`age` on TS's _own_ response. **Gates the Stage 3a/3b split** — see [§7](#7-deferred-work-specified-not-scheduled). **Step C — measure the hold directly (1 day + a measurement window).** The hold's cost is literally the duration of one `.await`: `collect_stream_auction` at -[publisher.rs:793](../../../crates/trusted-server-core/src/publisher.rs#L793), plus the +`publisher.rs:793`, plus the two EOF variants in `hold_finish_ready_segments` and `hold_finish_tail_segments`. Two `Instant`s around it yield **`hold_wait_ms`** — the number this entire document is arguing about, measured rather than modelled. @@ -182,7 +182,7 @@ browser harness when Stage 1 is actually scheduled. ## 4. Stage 0 — the only build item recommended now Stop bypassing the read-through cache on ad-eligible navigations -([publisher.rs:2867](../../../crates/trusted-server-core/src/publisher.rs#L2867)). +(`publisher.rs:2867`). **Ship it as an operator flag, not a deletion.** Add `publisher.bypass_origin_cache`, defaulting to today's behaviour, in the same release as @@ -191,7 +191,9 @@ the Step C instrumentation. Then turn it off with `ts config push`. The diff is slightly larger than deleting a line, and that is the point. The risk being gated here is **cache poisoning** — serving one representation in response to a request for another. For that class of failure, rollback speed dominates diff size: a config push -reverts in seconds, a release does not. The flag also buys an A/B on a byte-identical +reverts the read path in seconds where a release does not — but a config push **evicts +nothing**, so full rollback is flip, then purge or roll a versioned key namespace, then +observe past the origin TTL. The flag also buys an A/B on a byte-identical build, removing build difference as a confound in the very measurement this depends on, and allows flipping for a tester-cookie population before all traffic. @@ -203,7 +205,7 @@ its branch. A temporary flag left in place becomes permanent configuration surfa Two regression signals, both checked before the win is: - **`unexpected_origin_304` abandonment rate.** That reason - ([publisher.rs:2894-2916](../../../crates/trusted-server-core/src/publisher.rs#L2894-L2916), + (`publisher.rs:2894-2916`, emitted via `emit_abandoned_auction` at `:2360`) exists precisely because the ad-stack path refuses cached and conditional origin responses. Re-enabling the cache is what could revive it. **Any non-zero rate is a rollback signal** — it means a 304 is reaching @@ -214,7 +216,7 @@ Two regression signals, both checked before the win is: performance regression. **Why it is safe in principle.** The conditional-header strip runs 34 lines earlier -under the same gate ([publisher.rs:2832-2836](../../../crates/trusted-server-core/src/publisher.rs#L2832), +under the same gate (`publisher.rs:2832-2836`, which also strips `Range`/`If-Range`), so the request already reaches the cache unconditional and a HIT returns a full body. [The 304-prevention design](./2026-07-22-ssat-root-document-304-prevention-design.md) added the bypass as belt-and-braces and listed the TTFB cost under its own Risks. The @@ -222,7 +224,7 @@ strip alone satisfies its invariant. **But it carries a risk that design never considered — and this is the blocking precondition.** RSC fetches are not navigations -([is_navigation_request](../../../crates/trusted-server-core/src/http_util.rs#L73-L98) +(`is_navigation_request` requires `Sec-Fetch-Dest: document`), so they never set the bypass and **already flow through the readthrough cache**, while HTML navigations are `PASS`. Removing the bypass puts both representations under one cache key. #1009 states the origin varies on @@ -233,7 +235,7 @@ cache can serve a flight payload to an HTML navigation. The classification is also not airtight: `is_navigation_request` falls back to the `Accept` header when Fetch Metadata is absent, and its own comment warns _"this path is weaker — `fetch()` can set Accept: text/html"_ -([http_util.rs:84-88](../../../crates/trusted-server-core/src/http_util.rs#L84)). +(`http_util.rs:84-88`). **A FAIL is not merely a Stage 0 blocker — it is a live production defect.** RSC fetches already transit the read-through cache today, because they never set the bypass. If the @@ -285,13 +287,13 @@ The hold is load-bearing for something other than latency. The invariant is: > `ad_bids_state` must be `Some(..)` when `lol_html` processes the `` end tag. -The end-tag handler ([html_processor.rs:381-395](../../../crates/trusted-server-core/src/html_processor.rs#L381-L395)) +The end-tag handler (`html_processor.rs:381-395`) locks that mutex once and falls back to `build_empty_bids_script()` on `None`. **Removing the hold without relocating collection renders a normal page with `tsjs.bids = {}` and no server-side ads** — no error, no non-2xx, no ERROR log. On Axum, Cloudflare, and Spin the loss is fully silent: -[publisher.rs:2248](../../../crates/trusted-server-core/src/publisher.rs#L2248) holds a +`publisher.rs:2248` holds a bare `Option` with no guard, so not even a drop warning fires. **The SSPs are billed regardless.** @@ -304,14 +306,14 @@ fill cannot be the canary — see [§7](#7-deferred-work-specified-not-scheduled ### 6.1 Corrections to #1009's premises -| # | #1009 states | Verified against `cfb98f4` | -| --- | -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | Two per-user injection seams | `tsjs.adSlots` is per-URL — [build_slot_json](../../../crates/trusted-server-core/src/publisher.rs#L3501-L3525) emits config- and path-derived fields only. **One per-user hole.** | -| 2 | Identity off-inline is a prerequisite | Privacy net is cookie-gated and returning navs set no cookie ([ec/finalize.rs:86-94](../../../crates/trusted-server-core/src/ec/finalize.rs#L86-L94)). **First-visit only.** | -| 3 | Stamp at `:2882-2888` | [`:2945-2963`](../../../crates/trusted-server-core/src/publisher.rs#L2945-L2963), `private, no-store`, also removing `ETag`/`Last-Modified`/four CDN headers. **Seven headers.** | -| 4 | Three cacheability killers | Two more: `bypass_cache` and the [304→502 guard](../../../crates/trusted-server-core/src/publisher.rs#L2894-L2916). **The bypass is the cost.** | -| 5 | Two `!Send` pipelines is the hard part | `?Send` already pervasive. **Not the obstacle.** | -| 6 | Goal: root as a shared Fastly HIT | Nothing caches TS's own response on Compute; the A/B's `x-cache` is the **backend readthrough** cache. **Reframes the goal.** | +| # | #1009 states | Verified against `cfb98f4` | +| --- | -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Two per-user injection seams | `tsjs.adSlots` is per-URL — `build_slot_json` emits config- and path-derived fields only. **One per-user hole.** | +| 2 | Identity off-inline is a prerequisite | Privacy net is cookie-gated and returning navs set no cookie (`ec/finalize.rs:86-94`). **First-visit only** — but note the corollary: because returning navigations set no cookie, that net never fires for them and is **not** a backstop against shared-caching a per-user response. | +| 3 | Stamp at `:2882-2888` | `:2945-2963`, `private, no-store`, also removing `ETag`/`Last-Modified`/four CDN headers. **Seven headers.** | +| 4 | Three cacheability killers | Two more: `bypass_cache` and the `304→502 guard`. **The bypass is the cost.** | +| 5 | Two `!Send` pipelines is the hard part | `?Send` already pervasive. **Not the obstacle.** | +| 6 | Goal: root as a shared Fastly HIT | Nothing caches TS's own response on Compute; the A/B's `x-cache` is the **backend readthrough** cache. **Reframes the goal.** | Rows on #1009's `esi` compatibility check (holds), its drifted line numbers, and its two broken `#1`/`#3` cross-references are in [Appendix A](#appendix-a--full-1009-correction-table). @@ -327,9 +329,9 @@ argument for client-fill, which the issue then declines in favour of ESI. It was wrong, and the correction matters.** The hold does not key off `lol_html` at all. `BodyCloseHoldBuffer::push` -([publisher.rs:2190-2202](../../../crates/trusted-server-core/src/publisher.rs#L2190)) +(`publisher.rs:2190-2202`) scans the **decoded origin input** for `` while the auction rides alongside transfer" — is **inert on a Next.js publisher**. Every `step.ready` yields empty bytes. That comment is misleading on exactly the publisher @@ -481,7 +483,7 @@ Secondary wins: removes the duplicated per-codec decoder/encoder wiring, six com imports, and the non-parser-context ` Date: Mon, 10 Aug 2026 16:06:46 +0530 Subject: [PATCH 171/395] Reconcile the #1009 documents with the request-neutrality correction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six blockers from review, all verified against the source before fixing. The three-verdict Stage 0 gate was only half propagated. The findings template still offered PASS/FAIL and routed PASS straight to the flip, and the spec still approved Stage 0 on the Vary check alone. Both now use FINAL PASS / PROVISIONAL PASS / FAIL, and Task 5a is titled for FINAL PASS so the gate cannot be read past. The spec contradicted the spike on request-neutrality, which would have recreated the leakage bug the spike exists to avoid. It still described adSlots as per-URL, kept it in the template, and drew two markers. New section 6.7 gives the rule: content is per-URL, presence is gated on should_run_ad_stack and is therefore per-request, so it must live in the fragment. The correction-table row, the pipeline diagram, the disposition table and the appendix all point at it. The Core Cache example still would not compile and mishandled stale entries. It called Found::to_body, which does not exist — the accessor is to_stream and it is fallible. Worse, it tested found() before must_insert_or_update(), but a stale entry sets both: that ordering serves stale bytes and never fulfils the update obligation, leaving concurrent waiters blocked. Reordered, with abandon plus cancel_insert_or_update on transform failure and an explicit note that the stale state machine is the caller's to write. The finalization order was impossible. The plan streamed ESI output into the client body while claiming EC, geo and privacy headers finalize afterwards; streaming responses on this adapter commit headers first and then pipe chunks. The invariant is now stated the only way it can work: finalize every header, including an unconditional private/no-store, before any body byte is written. The decision rule adopted A3 on the metric the same document forbids. A2 and A3 serve the same template, so root TTFB is near-identical by construction. The rule now turns on bids-ready, adInit fire and first attributed creative paint, with root TTFB kept only as a non-regression guard. Added a request-scoped arm allocator, since a global setting yields sequential blocks and confounds arm with time of day and cache warmth. Operational: Stage 0's rollback pointed at Core Cache surrogate keys, which belong to the transformed-template cache the spike builds and have no effect on the HTTP read-through cache Stage 0 turns on. Purging that needs origin-supplied keys or the HTTP cache's own surface, and until one exists the rollback is waiting out the origin TTL — now recorded as an accepted risk rather than a discovery during an incident. --- ...2026-08-08-1009-measurement-and-stage-0.md | 40 ++++-- .../2026-08-10-1009-esi-validation-spike.md | 95 ++++++++++---- ...08-esi-cacheable-root-validation-design.md | 122 +++++++++++------- 3 files changed, 177 insertions(+), 80 deletions(-) diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md index 6298ca2c6..7a6d79a89 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md @@ -237,15 +237,22 @@ Create `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md`: **Cookie / auth exposure:** bodies differ by cookie? `Vary: Cookie` present? origin `Set-Cookie` on a shared-cacheable response? `Authorization`-gated responses cacheable? -**Verdict:** PASS / FAIL +**Verdict:** FINAL PASS / PROVISIONAL PASS / FAIL -PASS = `Vary` names every request header the origin varies on (`RSC`, any `Next-Router-*` -or experiment header whose value changed the body, **and `Cookie` if bodies differ by -cookie**), and no `Set-Cookie` rides a shared-cacheable response. -FAIL = any of the above is unmet. +`FINAL PASS` = `Vary` names every request header the origin varies on (`RSC`, any +`Next-Router-*` or experiment header whose value changed the body, **and `Cookie` if +bodies differ by cookie**), no `Set-Cookie` rides a shared-cacheable response, **and** all +five conditions in Task 5's gate are recorded — a real authenticated session cookie, Basic +Auth through TS, the experiment variant, representative routes, and cached-hit +slot/render attribution. -**Consequence:** PASS → Task 5a (flip the flag). FAIL → Task 5b (cache-key -discriminator). See spec §4. +`PROVISIONAL PASS` = the `Vary` and cookie checks hold, but one or more of those five is +untested. **Not a release gate.** A first pass lands here. + +`FAIL` = any `Vary` or `Set-Cookie` criterion is unmet. + +**Consequence:** `FINAL PASS` → Task 5a (flip the flag). `PROVISIONAL PASS` → close the +gaps before Task 5 starts. `FAIL` → Task 5b (cache-key discriminator). See spec §4. **A FAIL is also a live production defect, not only a Stage 0 blocker.** RSC fetches are not navigations, so they never set the bypass and **already transit the read-through @@ -778,7 +785,7 @@ already-deployed build**. No second release, and rollback is another config push than a revert. That matters here specifically: the failure mode this gates on is cache poisoning, where minutes of exposure are worse than a slow rollout. -### Task 5a: flip the flag (Task 1 verdict = PASS) +### Task 5a: flip the flag (Task 1 verdict = `FINAL PASS`) **Files:** @@ -896,11 +903,18 @@ expire. The origin's `max-age=60` bounds that, but does not remove it. Full rollback: 1. Push `bypass_origin_cache = true`. -2. Purge. `fastly::http::purge::purge_surrogate_key` runs inside Compute, with keys - attached at insert via `InsertBuilder::surrogate_keys`; alternatively roll a versioned - cache-key namespace. **Neither is wired today** — if the flip ships before one exists, - the rollback story is "wait out the TTL," and that must be an accepted risk rather - than an unnoticed one. +2. Purge — **and note this is C1, not C2.** `InsertBuilder::surrogate_keys` belongs to + the Core Cache API and applies to the transformed-template cache the ESI spike builds. + It has no effect on the HTTP read-through cache that Stage 0 turns on. Purging C1 + requires either surrogate keys the **origin** supplies on its responses, or the HTTP + cache's own request/candidate surrogate-key surface. Confirm which is available before + relying on it. + + **Neither is wired today.** If the flip ships before one exists, the rollback story is + "wait out the origin TTL" — roughly a minute, per the Step A findings. That is + survivable, but it must be an accepted risk recorded before the flip rather than a + discovery during an incident. + 3. Observe past the origin TTL before declaring the incident closed. - [ ] **Step 5: Run the full suite across every adapter** diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index 770cb05cc..befae1761 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -213,7 +213,18 @@ STALE / BYPASS from the transaction outcome. Record the serving POP alongside. A that mixes cold-MISS and warm-HIT requests is meaningless, and arms cannot be compared unless the mix is known — per tier. -- [ ] **Step 4: Define the sample plan before collecting anything** +- [ ] **Step 4: Build a request-scoped arm allocator** + +`AssemblyMode` as specified in Task 3 is a **global** setting, but the sample plan below +requires randomized, non-sequential allocation. A global flip gives sequential blocks +instead, which confounds arm with time of day, cache warmth, and traffic mix. + +Allocate per request: hash the lineage ID into buckets, or key off the tester cookie. +The global setting stays as the kill switch and as the way to force a single arm; the +allocator is what the experiment actually uses. Record the assigned arm on every log line +and every telemetry row. + +- [ ] **Step 5: Define the sample plan before collecting anything** Write all of this into the findings document **before** the first measurement, and treat it as fixed: @@ -337,27 +348,48 @@ use fastly::cache::core::{Transaction, CacheKey}; let tx = Transaction::lookup(CacheKey::from(key_bytes)).execute()?; -let template: Body = if let Some(found) = tx.found() { - found.to_body() // C2 HIT — skip origin fetch and transform -} else if tx.must_insert_or_update() { - // C2 MISS. Fetch origin, transform, insert, and read our own bytes back in one - // pass: execute_and_stream_back gives both the write handle and a readable Found. - let (mut writer, found) = tx - .insert(template_ttl) - .surrogate_keys(["ts-template", &url_surrogate_key]) // chained, not discarded - .user_metadata(metadata_envelope) // see below - .execute_and_stream_back()?; - stream_lol_html_output_into(&mut writer)?; - writer.finish()?; // REQUIRED - found.to_body() +// Order matters: a STALE entry sets BOTH found() and must_insert_or_update(). +// Testing found() first would serve the stale bytes and silently never fulfil the +// update obligation, leaving every concurrent waiter blocked until timeout. +let template: Body = if tx.must_insert_or_update() { + match transform_origin_into(&tx) { + Ok((writer, found)) => { + writer.finish()?; // REQUIRED — without it the object never completes + found.to_stream()? // fallible; there is no `to_body()` + } + Err(e) => { + // Do NOT finish() a partial template — it would be served to everyone + // until it expires. Abandon the writer, release the obligation so another + // client can try, and fall back to the untransformed path for this request. + writer.abandon(); + tx.cancel_insert_or_update()?; + return fallback_uncached(e); + } + } +} else if let Some(found) = tx.found() { + // Fresh hit. `is_usable()` and `is_stale()` are available if a stale-serve + // policy is wanted; the spike should start by treating stale as a miss. + found.to_stream()? // C2 HIT — skip origin fetch and transform } else { - unreachable!("transaction must either find or be obliged to insert") + unreachable!("a transaction is either obliged to insert or has found an item") }; ``` -`finish()` is not optional — without it the object never completes and its length stays -unknown. On any transform error, **cancel rather than finish**, or a partial template is -inserted and served to everyone until it expires. +Inside `transform_origin_into`, `execute_and_stream_back()` yields both handles at once: + +```rust +let (mut writer, found) = tx + .insert(template_ttl) + .surrogate_keys(["ts-template", &url_surrogate_key]) // chained, not discarded + .user_metadata(metadata_envelope) + .execute_and_stream_back()?; +``` + +**Decide the stale policy explicitly.** `Found::is_stale()` and `is_usable()` exist, and +`stale_while_revalidate` can be set at insert. Serving stale while revalidating is a real +option — but it is a state machine, and `cache::core` implements none of it for you. The +spike should start by treating stale as a miss and only add stale-serve if the numbers +justify it. **`cache::core` carries no HTTP semantics.** Status, headers, content encoding, and revalidation are all yours. Serialize what you need into `user_metadata` — at minimum the @@ -434,9 +466,23 @@ for the silent-empty-bids trap, which applies in full. themselves, which takes ownership away from the finalize / `ec_finalize` / apply-effects ordering. `process_stream(&mut self, src: impl BufRead, out: &mut impl Write, …)` keeps it. -Source is the C2 body. Sink is the response body on the way to the client — so EC cookie, -geo, and the privacy net still run **after** assembly. Confirm that ordering explicitly; -it is the difference between a correct response and a leaked one. +Source is the C2 body. Sink is the client response body. + +**The ordering an earlier draft described is impossible.** It said EC cookie, geo, and the +privacy net run _after_ assembly. They cannot: streaming responses on this adapter +**commit headers first and then pipe chunks** +(`adapter-fastly/src/main.rs`, `send_edgezero_response`). Once ESI starts writing, no +header can change. + +The correct invariant: + +> **Finalize every header before a single body byte is written** — EC `Set-Cookie`, geo +> suppression, and an unconditional `Cache-Control: private, no-store` — **then** stream +> the assembly with no further header mutation. + +That means `private, no-store` is set unconditionally up front rather than derived from +what the assembly turns out to contain. Deriving it after the fact is not available, and +assuming it was is how a per-user response ends up shared-cacheable. - [ ] **Step 2: Disable DCA explicitly and allowlist the dispatcher** @@ -566,8 +612,11 @@ Not a phase. Every one of these is a hard fail, independent of any performance r **Adopt ESI only if all three hold:** 1. Every Task 6 gate passes on A3. -2. A3 beats A2 on TTFB by a margin the reviewers ratify **before** collection — not - chosen after seeing the numbers. +2. A3 beats A2 on **bids-ready time, `adInit` fire time, and first TS-attributed creative + paint** — by a margin the reviewers ratify **before** collection, not chosen after + seeing the numbers. **Not root TTFB:** A2 and A3 serve the same C2 template, so their + root timings are near-identical by construction and a difference there would be noise. + Root TTFB is a non-regression guard only. 3. Render outcomes on A3 are non-inferior to A0. **Otherwise adopt A2 (client-fill)** if its gates pass and it beats A1. It is portable diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md index a3a35002a..446ff1727 100644 --- a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -306,14 +306,14 @@ fill cannot be the canary — see [§7](#7-deferred-work-specified-not-scheduled ### 6.1 Corrections to #1009's premises -| # | #1009 states | Verified against `cfb98f4` | -| --- | -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | Two per-user injection seams | `tsjs.adSlots` is per-URL — `build_slot_json` emits config- and path-derived fields only. **One per-user hole.** | -| 2 | Identity off-inline is a prerequisite | Privacy net is cookie-gated and returning navs set no cookie (`ec/finalize.rs:86-94`). **First-visit only** — but note the corollary: because returning navigations set no cookie, that net never fires for them and is **not** a backstop against shared-caching a per-user response. | -| 3 | Stamp at `:2882-2888` | `:2945-2963`, `private, no-store`, also removing `ETag`/`Last-Modified`/four CDN headers. **Seven headers.** | -| 4 | Three cacheability killers | Two more: `bypass_cache` and the `304→502 guard`. **The bypass is the cost.** | -| 5 | Two `!Send` pipelines is the hard part | `?Send` already pervasive. **Not the obstacle.** | -| 6 | Goal: root as a shared Fastly HIT | Nothing caches TS's own response on Compute; the A/B's `x-cache` is the **backend readthrough** cache. **Reframes the goal.** | +| # | #1009 states | Verified against `cfb98f4` | +| --- | -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Two per-user injection seams | Partly. `tsjs.adSlots` **content** is per-URL — `build_slot_json` emits config- and path-derived fields only. But its **presence** is gated on `should_run_ad_stack` (consent, bot, prefetch, kill switch), so it is request-dependent and **must not live in a shared template**. See §6.7. | +| 2 | Identity off-inline is a prerequisite | Privacy net is cookie-gated and returning navs set no cookie (`ec/finalize.rs:86-94`). **First-visit only** — but note the corollary: because returning navigations set no cookie, that net never fires for them and is **not** a backstop against shared-caching a per-user response. | +| 3 | Stamp at `:2882-2888` | `:2945-2963`, `private, no-store`, also removing `ETag`/`Last-Modified`/four CDN headers. **Seven headers.** | +| 4 | Three cacheability killers | Two more: `bypass_cache` and the `304→502 guard`. **The bypass is the cost.** | +| 5 | Two `!Send` pipelines is the hard part | `?Send` already pervasive. **Not the obstacle.** | +| 6 | Goal: root as a shared Fastly HIT | Nothing caches TS's own response on Compute; the A/B's `x-cache` is the **backend readthrough** cache. **Reframes the goal.** | Rows on #1009's `esi` compatibility check (holds), its drifted line numbers, and its two broken `#1`/`#3` cross-references are in [Appendix A](#appendix-a--full-1009-correction-table). @@ -416,9 +416,10 @@ tags that do not exist yet. The correct order: ``` origin → lol_html transform → fastly::cache::core → esi assemble → finalize → client - (emit esi:include at the (shared template, (per request, (EC cookie, - head + body-close seams, surrogate-keyed, fetch the geo, privacy - no per-user data) TS-chosen TTL) bids fragment) net) + (one unconditional marker (shared template, (per request, headers are + at the body-close seam, surrogate-keyed, fetch the finalized + no per-user data and no TS-chosen TTL) fragment) BEFORE the + request-dependent decisions) body streams ``` The push/pull mismatch that the earlier revision treated as a blocker is real but @@ -453,6 +454,39 @@ stale / shielding behaviour must run against a real Fastly test service. --- +### 6.7 What may and may not live in a shared template + +A correction to §6.1 row 1, and the constraint that governs any shared-template design. + +The original framing — "`adSlots` is per-URL, so there is one per-user hole, not two" — +is half right and dangerously so. `build_slot_json` really does emit only config- and +path-derived fields. But whether the script is emitted **at all** is gated on +`should_run_ad_stack` (`publisher.rs:2920-2927`), which is +`is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the _content_ is per-URL and the _presence_ is per-request. A shared object filled by +the first request would freeze that request's consent decision, bot classification, +prefetch status, and kill-switch state for every later reader. A consent-denied fill +serves a no-ads template to consenting users; a consenting fill serves ad markup to +someone who refused. + +**The rule for anything cached and shared:** + +| May live in the template | Must live in the per-request fragment | +| --------------------------------------- | ---------------------------------------------- | +| tsjs bundle script tag (content-hashed) | `tsjs.adSlots` — presence is request-gated | +| URL rewrites (per-host, in the key) | `tsjs.bids` | +| | GPT diagnostics bootstrap (cookie/query-gated) | +| | Integration head-inserts (request-scoped) | + +The test that catches this class is **byte-identity of the template across requests +differing in consent, bot classification, and prefetch status** — not an absence-of- +per-user-values scan, which the broken design would have passed. + +This applies to any shared-template work, ESI or client-fill alike. The +[spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) implements it. + --- ## 7. Deferred work, specified not scheduled @@ -600,7 +634,7 @@ Rows 1–6 are in [§6.1](#61-corrections-to-1009s-premises). The remainder: | `2832-2836` | strip `If-None-Match`, `If-Modified-Since`, `Range`, `If-Range` | **keep** — needed for any injection | | `2866` | `with_cache_bypass()` | **make operator-controlled** — [§4](#4-stage-0--the-only-build-item-recommended-now) | | `2894` | 304 → 502 guard | keep as safety net | -| `2920` | build `adSlots` | keep — per-URL | +| `2920` | build `adSlots` | keep, but see §6.7 — presence is request-dependent | | `2945` | strip cacheability | **replace** — Stage 3a | **Cache tiers.** T1 backend readthrough (already available; TS opts out). T2 TS-owned @@ -744,35 +778,35 @@ queued before it loads (#1009 Part 1) — not filed, should be. All pinned to `cfb98f4`. -| Concern | Location | -| -------------------------------------------- | ------------------------------------------------------------------------------- | -| Eligibility decision | `publisher.rs:2651`, `:2660` | -| `is_navigation_request` | `http_util.rs:73-98` | -| Auction dispatch (pre-origin, non-blocking) | `publisher.rs:2698-2760` | -| Auction overlap intent | `auction/orchestrator.rs:950-952` | -| Auction timeout resolution | `publisher.rs:2680-2684` (creative_opportunities, else auction.timeout_ms) | -| Conditional/range header strip | `publisher.rs:2832-2836` | -| Origin cache bypass | `publisher.rs:2866-2868` | -| Origin 304 → 502 guard | `publisher.rs:2894-2916` | -| `adSlots` build (per-URL) | `publisher.rs:2920`, `:3558-3577`, `:3501-3525` | -| Uncacheable stamp | `publisher.rs:2945-2963` | -| `` hold — sync / async / Fastly lazy | `publisher.rs:2235` / `:2109` / `:1318-1390` | -| Hold buffer | `publisher.rs:2177-2218` | -| Auction collect (HTML / non-HTML) | `publisher.rs:2431` (emits `:2456`) / `:2388` (emits `:2410`) | -| Abandonment emitter | `publisher.rs:2360` | -| Bids script build | `publisher.rs:3438-3491` | -| `/_ts/page-bids` | `publisher.rs:3611`, handler `:3723`, auction `:3903` | -| Injection seams (head / body-close) | `html_processor.rs:310-363` / `:381-395` | -| Post-processor buffering | `html_processor.rs:62-94` | -| Next.js post-processor registration | `integrations/nextjs/mod.rs:107` | -| Max buffered body (16 MB) | `settings.rs:77-79` | -| EC cookie issuance policy | `ec/finalize.rs:86-107` | -| Cookie-privacy net | `response_privacy.rs:20-61` | -| Geo response headers | `adapter-fastly/src/middleware.rs:194-200` | -| Cacheable-header precedent | `http_util.rs:294-311` | -| Management token lacks purge (wrong surface) | `adapter-fastly/src/management_api.rs:12` | -| In-process purge / surrogate keys | `fastly` 0.12.1 `cache::core`, `http::purge::purge_surrogate_key` | -| Client initial-ad gate | `js/lib/src/integrations/gpt/index.ts:536-555` | -| `adInit` bid application | `js/lib/src/integrations/gpt/index.ts:566`, `:652`, `:657-661` | -| Client SPA auction hook | `js/lib/src/integrations/gpt/index.ts:806`, `:859`, `:892-949` | -| Prior design that introduced the killers | `docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md` | +| Concern | Location | +| --------------------------------------------------------- | ------------------------------------------------------------------------------- | +| Eligibility decision | `publisher.rs:2651`, `:2660` | +| `is_navigation_request` | `http_util.rs:73-98` | +| Auction dispatch (pre-origin, non-blocking) | `publisher.rs:2698-2760` | +| Auction overlap intent | `auction/orchestrator.rs:950-952` | +| Auction timeout resolution | `publisher.rs:2680-2684` (creative_opportunities, else auction.timeout_ms) | +| Conditional/range header strip | `publisher.rs:2832-2836` | +| Origin cache bypass | `publisher.rs:2866-2868` | +| Origin 304 → 502 guard | `publisher.rs:2894-2916` | +| `adSlots` build (content per-URL, presence request-gated) | `publisher.rs:2920`, `:3558-3577`, `:3501-3525` | +| Uncacheable stamp | `publisher.rs:2945-2963` | +| `` hold — sync / async / Fastly lazy | `publisher.rs:2235` / `:2109` / `:1318-1390` | +| Hold buffer | `publisher.rs:2177-2218` | +| Auction collect (HTML / non-HTML) | `publisher.rs:2431` (emits `:2456`) / `:2388` (emits `:2410`) | +| Abandonment emitter | `publisher.rs:2360` | +| Bids script build | `publisher.rs:3438-3491` | +| `/_ts/page-bids` | `publisher.rs:3611`, handler `:3723`, auction `:3903` | +| Injection seams (head / body-close) | `html_processor.rs:310-363` / `:381-395` | +| Post-processor buffering | `html_processor.rs:62-94` | +| Next.js post-processor registration | `integrations/nextjs/mod.rs:107` | +| Max buffered body (16 MB) | `settings.rs:77-79` | +| EC cookie issuance policy | `ec/finalize.rs:86-107` | +| Cookie-privacy net | `response_privacy.rs:20-61` | +| Geo response headers | `adapter-fastly/src/middleware.rs:194-200` | +| Cacheable-header precedent | `http_util.rs:294-311` | +| Management token lacks purge (wrong surface) | `adapter-fastly/src/management_api.rs:12` | +| In-process purge / surrogate keys | `fastly` 0.12.1 `cache::core`, `http::purge::purge_surrogate_key` | +| Client initial-ad gate | `js/lib/src/integrations/gpt/index.ts:536-555` | +| `adInit` bid application | `js/lib/src/integrations/gpt/index.ts:566`, `:652`, `:657-661` | +| Client SPA auction hook | `js/lib/src/integrations/gpt/index.ts:806`, `:859`, `:892-949` | +| Prior design that introduced the killers | `docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md` | From cf204f08a45b6f6aef0be096d3469eca94fa2fc2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 16:35:32 +0530 Subject: [PATCH 172/395] Close the remaining #1009 cross-document contradictions and gate the sweep Four contradictions found by a mechanical sweep, all verified before fixing. Stage 0 was still summarized as gated only by the Vary check in the spec's decision table, and as reverting with a config push alone in the plan's Task 5 preamble. Both now point at FINAL PASS and at the full flip-purge-observe sequence. The findings still attached C1 rollback keys using InsertBuilder::surrogate_keys, which is the Core Cache API and keys the transformed-template cache the ESI spike would build. It has no effect on the HTTP read-through cache Stage 0 turns on. The spec's invalidation table had the same ambiguity in a row that read fine in section context and wrong when quoted; it is now split into explicit C1 and C2 rows. The Core Cache pseudocode still would not compile after the previous fix. The error arm referenced a writer only the success arm bound, and a helper taking &tx could not call Transaction::insert, which consumes self. Restructured so everything fallible that does not need the writer happens before insert, where cancel_insert_or_update is still reachable, and so finish and abandon are each reached from the arm that owns the writer. The safety gate still asserted privacy finalization runs after assembly, contradicting the streaming rule added directly above it. Headers commit before the body streams on this adapter, so the gate now asserts finalization happened first, including an unconditional private/no-store. The Task 3 file list still said markers go at two seams while the corrected design emits one unconditional body-close marker. Adds scripts/docs-invariants.py and makes it a named gate in both plans. Format and build are necessary but neither can see a claim corrected in one document and left standing in another, which is how every one of the last four review rounds found real defects. The checker is context-aware, since qualifying text usually wraps to an adjacent line, and it is meant to grow a check whenever a correction lands. --- ...2026-08-08-1009-measurement-and-stage-0.md | 28 +++++++- .../2026-08-08-1009-measurement-findings.md | 20 ++++-- .../2026-08-10-1009-esi-validation-spike.md | 68 ++++++++++++------- ...08-esi-cacheable-root-validation-design.md | 34 ++++++---- scripts/docs-invariants.py | 65 ++++++++++++++++++ 5 files changed, 166 insertions(+), 49 deletions(-) create mode 100755 scripts/docs-invariants.py diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md index 7a6d79a89..d4ab8ba27 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md @@ -25,6 +25,17 @@ for wasm-safe timing, `log` for instrumentation, Viceroy for adapter tests. **Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` (§3 Steps A/B/C and §4 Stage 0). Read §4 and §5 before starting Task 5. +**Before pushing, run three gates, not two:** + +```bash +cd docs && npm run format && npm run build && cd .. +python3 scripts/docs-invariants.py +``` + +`npm run build` is not optional — `format` passes on documents with dead links, and that +shipped a broken docs build on this branch once already. `docs-invariants.py` catches +cross-document contradictions, which neither of the other two can see. + **Two prettier gotchas, both hit while writing this plan.** CI gate 7 (`cd docs && npm run format`) runs `prettier --check` over all of `docs/`, so both bite. @@ -781,9 +792,15 @@ Any one of these unrecorded means the verdict stays `PROVISIONAL PASS` and Task not start. Because Task 3 made the bypass config-driven, Stage 0 ships as a **config change on an -already-deployed build**. No second release, and rollback is another config push rather -than a revert. That matters here specifically: the failure mode this gates on is cache -poisoning, where minutes of exposure are worse than a slow rollout. +already-deployed build** — no second release, and the read path reverts with another +config push rather than a revert. That matters here: the failure mode this gates on is +cache poisoning, where minutes of exposure are worse than a slow rollout. + +**But a config push is not a full rollback.** It stops HTML navigations reading from +cache; it evicts nothing already stored. See Step 4's rollback sequence — flip, then purge +or roll a versioned namespace, then observe past the origin TTL. Until a C1 purge path +exists, the tail is "wait out the origin TTL," and that must be an accepted, recorded +risk before the flip. ### Task 5a: flip the flag (Task 1 verdict = `FINAL PASS`) @@ -1025,5 +1042,10 @@ Named so nobody widens this plan mid-flight. All are specified in the spec. measuring the TTFB the publisher actually complained about; use it for before/after. - [ ] `unexpected_origin_304` rate is zero and representations are not mixed (Task 5a Step 8) — both checked **before** the win is claimed. +- [ ] **Cross-document invariants hold:** `python3 scripts/docs-invariants.py`. This is a + named gate, not a courtesy check. `npm run format` and `npm run build` catch + formatting and dead links; neither catches a claim corrected in one document and + left standing in another, which is the failure mode this document set has hit on + four separate review rounds. Add a check whenever a correction lands. - [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md index a125d957c..a092f6c52 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -137,12 +137,20 @@ continue to read — persist until they expire. Two mitigations, both real: - The origin's `max-age=60` bounds read-through exposure to roughly a minute. -- Purge is available in-process: `fastly::http::purge::purge_surrogate_key`, with keys - attached at insert via `InsertBuilder::surrogate_keys`. An earlier claim that TS had no - purge capability was wrong — it has no _wiring_, which is buildable. - -Rollback is therefore: flip the flag, **then** purge or roll a versioned cache-key -namespace, **then** observe past the origin TTL before declaring the incident closed. +- Purge exists in-process — `fastly::http::purge::purge_surrogate_key`. An earlier claim + that TS had no purge capability was wrong; it has no _wiring_, which is buildable. + +**But note which cache.** `InsertBuilder::surrogate_keys` belongs to the **Core Cache** +API and applies to the transformed-template cache the ESI spike would build (C2). It has +**no effect on the HTTP read-through cache** that Stage 0 turns on (C1). Purging C1 needs +surrogate keys the _origin_ supplies on its responses, or the HTTP cache's own +request/candidate surrogate-key surface. Confirm which is available before relying on it — +an earlier revision of this document conflated the two. + +Rollback is therefore: flip the flag, **then** purge C1 by whichever mechanism is actually +available (or roll a versioned key namespace), **then** observe past the origin TTL before +declaring the incident closed. With no C1 purge path wired, the tail is the TTL itself — +roughly a minute here, and a recorded risk rather than a surprise. ## Step B — consumers of TS's own response headers diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index befae1761..ea4137c6a 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -251,7 +251,7 @@ The core of the spike. Behind a flag, default off. **Files:** -- `crates/trusted-server-core/src/publisher.rs` — emit markers at the two seams +- `crates/trusted-server-core/src/publisher.rs` — emit **one** unconditional marker at the body-close seam (see Step 2; the head seam is not a template hole) - `crates/trusted-server-core/src/settings.rs` — the mode flag - `crates/trusted-server-adapter-fastly/src/` — the `cache::core` read/write @@ -352,38 +352,48 @@ let tx = Transaction::lookup(CacheKey::from(key_bytes)).execute()?; // Testing found() first would serve the stale bytes and silently never fulfil the // update obligation, leaving every concurrent waiter blocked until timeout. let template: Body = if tx.must_insert_or_update() { - match transform_origin_into(&tx) { - Ok((writer, found)) => { - writer.finish()?; // REQUIRED — without it the object never completes - found.to_stream()? // fallible; there is no `to_body()` + // Fetch and prepare BEFORE consuming `tx`. After `insert()` the transaction is + // gone and `cancel_insert_or_update()` is unreachable, so anything that can fail + // and does not need the writer belongs here. + let origin = match fetch_and_prepare_origin() { + Ok(origin) => origin, + Err(e) => { + tx.cancel_insert_or_update()?; // releases the obligation to a waiter + return fallback_uncached(e); + } + }; + + // `Transaction::insert(self)` consumes `tx` from this line on. + let (mut writer, found) = tx + .insert(template_ttl) + .surrogate_keys(["ts-template", &url_surrogate_key]) // chained, not discarded + .user_metadata(metadata_envelope) + .execute_and_stream_back()?; + + match stream_lol_html_output(origin, &mut writer) { + Ok(()) => { + writer.finish()?; // REQUIRED, and consumes `writer` + found.to_stream()? // fallible; there is no `to_body()` } Err(e) => { - // Do NOT finish() a partial template — it would be served to everyone - // until it expires. Abandon the writer, release the obligation so another - // client can try, and fall back to the untransformed path for this request. - writer.abandon(); - tx.cancel_insert_or_update()?; + // Also consumes `writer`, marking an unsuccessful end so no partial + // template is served. (A `StreamingBody` dropped without `finish()` is + // aborted anyway, but say it explicitly.) + writer.abandon()?; return fallback_uncached(e); } } } else if let Some(found) = tx.found() { - // Fresh hit. `is_usable()` and `is_stale()` are available if a stale-serve - // policy is wanted; the spike should start by treating stale as a miss. - found.to_stream()? // C2 HIT — skip origin fetch and transform + found.to_stream()? // C2 HIT — skip origin fetch and transform } else { unreachable!("a transaction is either obliged to insert or has found an item") }; ``` -Inside `transform_origin_into`, `execute_and_stream_back()` yields both handles at once: - -```rust -let (mut writer, found) = tx - .insert(template_ttl) - .surrogate_keys(["ts-template", &url_surrogate_key]) // chained, not discarded - .user_metadata(metadata_envelope) - .execute_and_stream_back()?; -``` +Two ownership rules this shape exists to respect, both of which an earlier draft broke: +`Transaction::insert(self)` **consumes** the transaction, so a helper taking `&tx` cannot +call it and `cancel_insert_or_update` is unreachable afterwards; and `finish`/`abandon` +each consume the writer, so neither can be referenced from an arm that did not bind it. **Decide the stale policy explicitly.** `Found::is_stale()` and `is_usable()` exist, and `stale_while_revalidate` can be set at insert. Serving stale while revalidating is a real @@ -582,9 +592,12 @@ Not a phase. Every one of these is a hard fail, independent of any performance r - [ ] **Request collapsing** works: concurrent cold requests transform once. - [ ] **DCA disabled**, verified by the injection test in Task 5 Step 2. - [ ] **Exactly one auction per pageview**, from `auction_events_raw`. -- [ ] **Cookie and privacy finalization still run** after assembly — EC `Set-Cookie` on - first visit, and the privacy net downgrading it. This is the ordering that ESI's - streaming mode makes easy to get wrong, since it drops `$add_header`. +- [ ] **Cookie and privacy finalization ran BEFORE assembly**, not after — EC + `Set-Cookie` on first visit, geo suppression, and an unconditional + `Cache-Control: private, no-store`. Headers commit before the body streams on this + adapter, so "finalize after assembly" is not available; asserting it that way is how + a per-user response ends up shared-cacheable. ESI's streaming mode dropping + `$add_header` is a consequence of the same constraint, not a separate hazard. - [ ] **Slot and bid attribution unchanged.** Same slots matched, same bids applied, same renders attributed. Use TS-attributed renders — the SSAT line item, non-empty `ts.bids`, `hb_adid` presence — **never slot fill**, which is blind to empty bids @@ -668,5 +681,10 @@ routes; N per arm; and the cache-tier mix. the answer. - [ ] Cleanup complete: flag resolved, C2 purged, synthetic endpoint removed, dependency landed or dropped. +- [ ] **Cross-document invariants hold:** `python3 scripts/docs-invariants.py`. This is a + named gate, not a courtesy check. `npm run format` and `npm run build` catch + formatting and dead links; neither catches a claim corrected in one document and + left standing in another, which is the failure mode this document set has hit on + four separate review rounds. Add a check whenever a correction lands. - [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md index 446ff1727..fda8b6e4c 100644 --- a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -17,9 +17,12 @@ > `insert(key, max_age).execute() -> StreamingBody` for arbitrary bytes, `lookup()` / > `found()` to read them back, and `Transaction` with `must_insert()` for request > collapsing. The two-stage design needs no separate KV or template service. -> - **Purge exists in-process.** `InsertBuilder::surrogate_keys([...])` attaches keys at -> insert; `fastly::http::purge::purge_surrogate_key` purges from inside Compute. The -> management-API token scope cited in the original is irrelevant to it. +> - **Purge exists in-process.** `fastly::http::purge::purge_surrogate_key` purges from +> inside Compute; the management-API token scope cited in the original is irrelevant to +> it. Note which cache, though: `InsertBuilder::surrogate_keys([...])` is the **Core +> Cache** API and keys the transformed-template cache (C2). It does **not** key the HTTP +> read-through cache (C1) that Stage 0 turns on — purging that needs origin-supplied +> keys or the HTTP cache's own surrogate-key surface. > - **The original pipeline ordering was backwards.** It said "order esi → lol*html, > never the reverse." `lol_html` \_emits* the `esi:include` tags, so ESI must run after > it. Correct order is in [§6.6](#66-the-esi-pipeline-corrected). @@ -59,12 +62,12 @@ ## 1. Decision requested -| # | Decision | Owner needed | -| --- | --------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | -| D1 | **ESI is feasible and unvalidated.** Validate it via [the spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), not by deferring it. | Eng + product | -| D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | -| D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to the `Vary` check in §3. | Eng | -| D4 | **Stages 1–2 queue behind the SSAT price defect and #938.** Stages 3b–5 unscheduled. | Product | +| # | Decision | Owner needed | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| D1 | **ESI is feasible and unvalidated.** Validate it via [the spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), not by deferring it. | Eng + product | +| D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | +| D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to a **`FINAL PASS`** in §3. A `Vary` check alone is a `PROVISIONAL PASS` and is not a release gate. Rollback needs a purge path, not only a config push. | Eng | +| D4 | **Stages 1–2 queue behind the SSAT price defect and #938.** Stages 3b–5 unscheduled. | Product | Rationale for D4 in [§8](#8-priority). Everything this document recommends _against_ doing is in [§7](#7-deferred-work-specified-not-scheduled), at deliberately lower @@ -429,12 +432,13 @@ what #1009 proposed in the first place. Mechanism, all present in the pinned `fastly` 0.12.1: -| Need | API | -| ----------------------- | ----------------------------------------------------------------------------------- | -| Store the template | `cache::core::insert(key, max_age).execute() -> StreamingBody` | -| Read it back | `cache::core::lookup(key)` → `found()` | -| Avoid a thundering herd | `cache::core::Transaction` — `must_insert()` / `must_insert_or_update()` | -| Invalidate | `InsertBuilder::surrogate_keys([...])` + `fastly::http::purge::purge_surrogate_key` | +| Need | API | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Store the template | `cache::core::insert(key, max_age).execute() -> StreamingBody` | +| Read it back | `cache::core::lookup(key)` → `found()` | +| Avoid a thundering herd | `cache::core::Transaction` — `must_insert()` / `must_insert_or_update()` | +| Invalidate **C2 only** | `InsertBuilder::surrogate_keys([...])` (Core Cache) + `fastly::http::purge::purge_surrogate_key`. Does **not** key C1 — see the row below. | +| Invalidate C1 | Origin-supplied surrogate keys, or the HTTP cache's own surrogate-key surface. Not the Core Cache API. | Purge runs **inside Compute**. The management-API token scope cited under [Stage 4](#7-deferred-work-specified-not-scheduled) governs a different surface and does diff --git a/scripts/docs-invariants.py b/scripts/docs-invariants.py new file mode 100755 index 000000000..5eb6eb3d8 --- /dev/null +++ b/scripts/docs-invariants.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Cross-document invariant sweep for the #1009 document set. + +`npm run format` and `npm run build` catch formatting and dead links. Neither +catches a claim corrected in one document and left standing in another, which is +the failure mode this set has hit repeatedly. + +Checks are context-aware: a hit is excused only if an allowlist pattern appears +within a window of lines around it, because qualifying text usually wraps. + +Exit 1 on any surviving hit. +""" +import re, sys, glob, os + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +FILES = sorted( + glob.glob(os.path.join(ROOT, "docs/superpowers/specs/2026-08-08-esi-*.md")) + + glob.glob(os.path.join(ROOT, "docs/superpowers/plans/2026-08-*1009*.md")) +) + +# (name, forbidden pattern, allowlist pattern or None, context window in lines) +CHECKS = [ + ("Stage 0 gate is three-verdict", + r"PASS / FAIL|verdict = PASS[^A-Za-z]|subject to the `Vary` check", + r"PROVISIONAL PASS / FAIL|FINAL PASS", 0), + ("Rollback is not described as config-only", + r"rollback is another config push rather", None, 0), + ("adSlots is not in the shared template", + r"two seams|adSlots[^.]*stays in the template|One per-user hole\.", None, 0), + ("Headers finalize before assembly", + r"finalization still run.{0,4} after assembly|run \*\*after\*\* assembly", None, 0), + ("No nonexistent Core Cache APIs", + r"(? Date: Mon, 10 Aug 2026 17:07:32 +0530 Subject: [PATCH 173/395] Fix stale summaries and rebuild the invariant checker so it cannot false-green MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The checker added in cf204f08 reported 8/8 green on documents that still contained the contradictions it claimed to check. That is worse than having no checker: it certifies bad state. Three causes, each now addressed. It matched literal phrases. The stale text said "two existing injection seams", the pattern looked for "two seams". Patterns are now semantic and tolerant of wording. It matched line by line, so any phrase wrapped across a line break was invisible. Files are now whitespace-normalized before matching, which is how the architecture arrows spanning several lines were being missed. It had no way to know it had stopped working. Every check now carries fixtures: strings that must trip it, and corrected strings that must not. The script exits 2 and refuses to report anything if its own fixtures fail. Writing them caught two of my patterns not firing at all — one defeated by markdown emphasis between "Verdict:" and "PASS", another by a sentence boundary. Proof rather than assertion: run against the cf204f08 tree, the new checker flags all five contradictions there, including the four this review named. The old checker reported that same tree green. The stale text itself. The spike's architecture summary still said two injection seams and ordered assemble before finalize. The spec still described the cheap curl as gating Stage 0, mapped the Vary result straight to a config push, and summarized rollback as config-only in the priority section. Its pipeline diagram contradicted its own caption — the caption said headers finalize first while the arrows still read assemble then finalize. That diagram is a good example of why literal matching failed and why diagrams need checking as prose does. Also disambiguated the Stage 4 note, which cited InsertBuilder::surrogate_keys without saying it keys C2 rather than the C1 read-through cache Stage 0 turns on. --- .../2026-08-10-1009-esi-validation-spike.md | 16 +- ...08-esi-cacheable-root-validation-design.md | 37 +-- scripts/docs-invariants.py | 236 ++++++++++++++---- 3 files changed, 218 insertions(+), 71 deletions(-) diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index ea4137c6a..0fb273585 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -8,11 +8,17 @@ ESI and client-fill against it, and produce a decision record that either adopts ESI, adopts client-fill, or rejects both — with the Fastly-only maintenance cost priced in. -**Architecture:** `origin → lol_html transform → fastly::cache::core → assemble → finalize`. -The transform emits `esi:include` markers at the two existing injection seams instead of -inlining per-user data. The cached object is a shared template with no per-user bytes. -Assembly is either the `esi` crate (edge) or a client fetch of `/_ts/page-bids` (browser), -selected per request by config so both can be measured on one build. +**Architecture:** +`origin → lol_html transform → fastly::cache::core → finalize headers → stream assembly`. + +Headers finalize **before** assembly, not after — streaming responses on this adapter +commit headers first and then pipe chunks, so nothing can be set once assembly starts. + +The transform emits **one unconditional marker at the body-close seam**. Not two: the +head seam is not a template hole, because `tsjs.adSlots` presence is request-gated +(Task 3 Step 2). The cached object is a shared template with no per-user bytes and no +request-dependent decisions. Assembly is either the `esi` crate (edge) or a client fetch +(browser), selected per request by the arm allocator so both are measured on one build. **Tech Stack:** Rust 2024, `wasm32-wasip1`, `fastly` 0.12.1 (`cache::core`, `http::purge`), `esi` 0.7, `lol_html`, a real Fastly test service for cache behaviour. diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md index fda8b6e4c..39a65dac5 100644 --- a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -118,9 +118,12 @@ supply it: they compare cached fetches against each other, not against an origin Three checks, ordered cheapest-first. Each needs a named owner before starting. -**Step A — origin `Vary` check (minutes).** `curl` the origin with and without `RSC`, -`Next-Router-*`, and the experiment header; inspect the `Vary` response header. -**Gates Stage 0**, the only build item recommended now. Do this first because it is the +**Step A — origin `Vary` and cookie check (minutes for the first pass).** `curl` the +origin with and without `RSC`, `Next-Router-*`, and the experiment header; inspect `Vary`, +`Cache-Control`, and `Set-Cookie`. **This first pass yields a `PROVISIONAL PASS` only** — +it is not what gates the flip. A `FINAL PASS` additionally requires a real authenticated +session, Basic Auth through TS, the experiment variant, representative routes, and +cached-hit render attribution. Do the cheap pass first because it is the cheapest thing that unblocks anything. **Step B — what consumes TS's own response headers (under a day).** Request a TS-served @@ -264,7 +267,8 @@ three are a larger class than the RSC split: So Step A must capture `Cache-Control` and `Set-Cookie` too, and repeat each request with and without a session cookie. Same minutes of work; closes the bigger hole. -**Two effort branches, and Step A decides which:** +**Two effort branches, and Step A's `Vary` result decides which** — note this selects the +_shape_ of Stage 0, while the `FINAL PASS` conditions decide _whether it ships at all_: | Step A result | Stage 0 is… | Effort | | ---------------------- | --------------------------------------------- | ------ | @@ -418,11 +422,13 @@ That is backwards. `lol_html` is what \_emits* the `esi:include` tags; ESI canno tags that do not exist yet. The correct order: ``` -origin → lol_html transform → fastly::cache::core → esi assemble → finalize → client - (one unconditional marker (shared template, (per request, headers are - at the body-close seam, surrogate-keyed, fetch the finalized - no per-user data and no TS-chosen TTL) fragment) BEFORE the - request-dependent decisions) body streams +origin → lol_html transform → fastly::cache::core → finalize headers → stream esi assembly → client + (one unconditional marker (shared template, (EC cookie, geo, (per request, + at the body-close seam; surrogate-keyed, unconditional fetch the + the head seam is NOT a TS-chosen TTL) private/no-store) fragment) + hole — adSlots presence + is request-gated, §6.7) nothing may change + after this point ``` The push/pull mismatch that the earlier revision treated as a blocker is real but @@ -545,9 +551,10 @@ until a topology change. **Stage 4 — purge wiring.** Not sized. Prerequisite for a TS-owned cache. TS today emits no `Surrogate-Key` and holds a management token scoped without purge — but that token is -the wrong surface: `InsertBuilder::surrogate_keys` and -`fastly::http::purge::purge_surrogate_key` are both in the pinned SDK and purge runs -inside Compute ([§6.6](#66-the-esi-pipeline-corrected)). This is **missing wiring, not a +the wrong surface: `InsertBuilder::surrogate_keys` is the Core Cache API and +`fastly::http::purge::purge_surrogate_key` runs inside Compute, both in the pinned SDK +([§6.6](#66-the-esi-pipeline-corrected)). Note those key **C2**, the TS-owned template +cache — not the HTTP read-through cache C1. This is **missing wiring, not a platform limit.** Until it exists, any TS-owned cache is TTL-only and a config push takes up to one TTL to take effect. @@ -590,8 +597,10 @@ comment that it is _"rejected until an access-log emitter is wired"_). That is a follow-on, not part of this work. **Stage 0 next**, shipped as the operator flag in §4 rather than a deletion. Gated on a -`curl`, reverses an origin-load cost the prior design explicitly accepted, and rolls back -with a config push. Closer to a defect fix than an optimization — TS opted out of a cache +`FINAL PASS` — a `curl` alone yields only a `PROVISIONAL PASS`. It reverses an origin-load +cost the prior design explicitly accepted; the read path reverts with a config push, but +full rollback also needs a C1 purge path or waiting out the origin TTL. Closer to a defect +fix than an optimization — TS opted out of a cache it did not need to opt out of. **Stages 1–2 queue behind the correctness defects.** Their failure mode is silent diff --git a/scripts/docs-invariants.py b/scripts/docs-invariants.py index 5eb6eb3d8..562198cb3 100755 --- a/scripts/docs-invariants.py +++ b/scripts/docs-invariants.py @@ -1,65 +1,197 @@ #!/usr/bin/env python3 """Cross-document invariant sweep for the #1009 document set. -`npm run format` and `npm run build` catch formatting and dead links. Neither +`npm run format` catches formatting. `npm run build` catches dead links. Neither catches a claim corrected in one document and left standing in another, which is -the failure mode this set has hit repeatedly. +how every review round on this branch has found real defects. -Checks are context-aware: a hit is excused only if an allowlist pattern appears -within a window of lines around it, because qualifying text usually wraps. +DESIGN NOTES — a previous version of this checker reported 8/8 green on +documents that still contained the exact contradictions it claimed to check. +Three things caused that, and each is addressed here: -Exit 1 on any surviving hit. +1. It matched literal phrases ("two seams") that the stale text did not use + ("two existing injection seams"). Patterns are now semantic and tolerant. +2. It matched line by line, so any phrase wrapped across a line break was + invisible. Text is now whitespace-normalized per file before matching. +3. It had no way to know it had stopped working. Every check now carries + `must_flag` fixtures — strings that MUST trip it — and the script fails if + any fixture is not caught. A check that cannot fail is treated as broken. + +A false positive costs a minute. A false green costs a merge. """ -import re, sys, glob, os +import re +import sys +import glob +import os ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") -FILES = sorted( - glob.glob(os.path.join(ROOT, "docs/superpowers/specs/2026-08-08-esi-*.md")) - + glob.glob(os.path.join(ROOT, "docs/superpowers/plans/2026-08-*1009*.md")) -) -# (name, forbidden pattern, allowlist pattern or None, context window in lines) + +def targets(): + return sorted( + glob.glob(os.path.join(ROOT, "docs/superpowers/specs/2026-08-08-esi-*.md")) + + glob.glob(os.path.join(ROOT, "docs/superpowers/plans/2026-08-*1009*.md")) + ) + + +def normalize(text): + """Collapse whitespace so wrapped phrases match, and strip table padding.""" + text = re.sub(r"\s*\n\s*", " ", text) + text = re.sub(r"[ \t]{2,}", " ", text) + return text + + +class Check: + def __init__(self, name, bad, allow=None, must_flag=(), must_pass=()): + self.name = name + self.bad = re.compile(bad, re.I) + self.allow = re.compile(allow, re.I) if allow else None + self.must_flag = must_flag # strings this check MUST catch + self.must_pass = must_pass # corrected strings it must NOT catch + + def hits(self, normalized): + out = [] + for m in self.bad.finditer(normalized): + lo, hi = max(0, m.start() - 240), min(len(normalized), m.end() + 240) + if self.allow and self.allow.search(normalized[lo:hi]): + continue + out.append(normalized[max(0, m.start() - 60):m.end() + 60].strip()) + return out + + CHECKS = [ - ("Stage 0 gate is three-verdict", - r"PASS / FAIL|verdict = PASS[^A-Za-z]|subject to the `Vary` check", - r"PROVISIONAL PASS / FAIL|FINAL PASS", 0), - ("Rollback is not described as config-only", - r"rollback is another config push rather", None, 0), - ("adSlots is not in the shared template", - r"two seams|adSlots[^.]*stays in the template|One per-user hole\.", None, 0), - ("Headers finalize before assembly", - r"finalization still run.{0,4} after assembly|run \*\*after\*\* assembly", None, 0), - ("No nonexistent Core Cache APIs", - r"(?)\s*finaliz" + r"|finaliz\w*[^.]{0,40}(runs?|still run)[^.]{0,20}after\s+assembly", + must_flag=[ + "`origin → lol_html transform → fastly::cache::core → assemble → finalize`", + "esi assemble → finalize → client", + "Cookie and privacy finalization still run after assembly", + ], + must_pass=[ + "fastly::cache::core → finalize headers → stream assembly", + "Cookie and privacy finalization ran BEFORE assembly, not after", + ], + ), + Check( + "No nonexistent Core Cache APIs", + r"(? Date: Mon, 10 Aug 2026 17:50:31 +0530 Subject: [PATCH 174/395] Make the #1009 docs invariant gate fail closed --- scripts/docs-invariants.py | 149 ++++++++++++++++++++++++++++++------- 1 file changed, 122 insertions(+), 27 deletions(-) diff --git a/scripts/docs-invariants.py b/scripts/docs-invariants.py index 562198cb3..e88fb81b2 100755 --- a/scripts/docs-invariants.py +++ b/scripts/docs-invariants.py @@ -16,6 +16,13 @@ 3. It had no way to know it had stopped working. Every check now carries `must_flag` fixtures — strings that MUST trip it — and the script fails if any fixture is not caught. A check that cannot fail is treated as broken. +4. It let unrelated nearby correction language excuse a violation. Generic + allow-windows are gone; the one qualified check must span the exact API + occurrence it excuses. + +The four promised input documents and both fixture directions are mandatory. +Missing inputs, unreadable inputs, or an empty fixture side make the checker +itself fail with exit 2 before it reports document results. A false positive costs a minute. A false green costs a merge. """ @@ -26,6 +33,15 @@ ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +REQUIRED_TARGETS = frozenset( + { + "docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md", + "docs/superpowers/plans/2026-08-08-1009-measurement-findings.md", + "docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md", + "docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md", + } +) + def targets(): return sorted( @@ -34,26 +50,64 @@ def targets(): ) +def target_errors(paths): + """Return setup errors when any document this gate promises to scan is absent.""" + present = { + os.path.relpath(path, ROOT).replace(os.sep, "/") + for path in paths + } + return [ + f" missing required document: {path}" + for path in sorted(REQUIRED_TARGETS - present) + ] + + +def load_documents(paths): + documents = {} + errors = [] + for path in paths: + try: + with open(path, encoding="utf-8") as source: + documents[path] = normalize(source.read()) + except (OSError, UnicodeError) as error: + errors.append(f" cannot read {os.path.relpath(path, ROOT)}: {error}") + return documents, errors + + def normalize(text): - """Collapse whitespace so wrapped phrases match, and strip table padding.""" + """Collapse wraps while removing Markdown blockquote continuation markers.""" + text = re.sub(r"(?m)^\s*>\s?", "", text) text = re.sub(r"\s*\n\s*", " ", text) text = re.sub(r"[ \t]{2,}", " ", text) return text class Check: - def __init__(self, name, bad, allow=None, must_flag=(), must_pass=()): + def __init__(self, name, bad, must_flag=(), must_pass=()): self.name = name self.bad = re.compile(bad, re.I) - self.allow = re.compile(allow, re.I) if allow else None self.must_flag = must_flag # strings this check MUST catch self.must_pass = must_pass # corrected strings it must NOT catch def hits(self, normalized): out = [] for m in self.bad.finditer(normalized): - lo, hi = max(0, m.start() - 240), min(len(normalized), m.end() + 240) - if self.allow and self.allow.search(normalized[lo:hi]): + out.append(normalized[max(0, m.start() - 60):m.end() + 60].strip()) + return out + + +class QualifiedOccurrenceCheck(Check): + """Flag each occurrence unless its own local clause states the required relation.""" + + def __init__(self, name, occurrence, qualified, must_flag=(), must_pass=()): + super().__init__(name, occurrence, must_flag=must_flag, must_pass=must_pass) + self.qualified = re.compile(qualified, re.I) + + def hits(self, normalized): + out = [] + qualified_spans = [match.span() for match in self.qualified.finditer(normalized)] + for m in self.bad.finditer(normalized): + if any(lo <= m.start() and m.end() <= hi for lo, hi in qualified_spans): continue out.append(normalized[max(0, m.start() - 60):m.end() + 60].strip()) return out @@ -64,26 +118,32 @@ def hits(self, normalized): "Stage 0 gates on FINAL PASS, not a bare Vary check", # `[*_ ]*` absorbs markdown emphasis; `gated on a curl` is its own shape # because the sentence boundary defeats a proximity match to "Stage 0". - r"gates?[*_ ]+stage[*_ ]*0(?![^.]{0,140}final pass)" + r"gates?[*_ ]+stage[*_ ]*0" + r"(?!\s*only\s+(?:once|after|when)\s+(?:an?\s+)?[`*_]*final\s+pass[`*_]*" + r"(?:\s+is)?\s+(?:recorded|obtained|achieved)\b)" r"|verdict[:*_ ]+pass[*_ ]*/[*_ ]*fail" r"|gated\s+on\s+a\s+.?curl", - allow=r"final pass|provisional pass", must_flag=[ "inspect the `Vary` response header. **Gates Stage 0**, the only build item", "**Verdict:** PASS / FAIL", "Stage 0 next, shipped as the operator flag. Gated on a `curl`, reverses an origin-load cost", + "Inspect Vary. **Gates Stage 0** immediately. A separate sentence says FINAL PASS is required.", + "Gates Stage 0 immediately; FINAL PASS gates deployment later.", + "The Vary result gates Stage 0 only on a successful curl, while FINAL PASS gates production rollout.", ], must_pass=[ "Gates Stage 0 only once a FINAL PASS is recorded", + "Gates Stage 0 only after a FINAL PASS is recorded", "**Verdict:** FINAL PASS / PROVISIONAL PASS / FAIL", ], ), Check( "Rollback is not described as config-only", - r"rolls?\s+back\s+with\s+a\s+config\s+push(?![^.]{0,160}(purge|ttl))" + r"rolls?\s+back\s+with\s+a\s+config\s+push" r"|rollback\s+is\s+another\s+config\s+push\s+rather", must_flag=[ "reverses an origin-load cost, and rolls back with a config push. Closer to a defect fix", + "This rolls back with a config push, explicitly without a purge or TTL wait.", ], must_pass=[ "the read path reverts with a config push, but full rollback also needs a C1 purge path", @@ -91,16 +151,18 @@ def hits(self, normalized): ), Check( "Template has one marker, not two seams", - r"(two|both)\s+(existing\s+)?(injection\s+)?seams" - r"|markers?\s+at\s+the\s+two\s+seams" + r"\b(?:emit|emits|use|uses|place|places|insert|inserts)\b[^.]{0,100}" + r"(?:two|both)\s+(?:existing\s+)?(?:injection\s+)?seams" + r"|markers?\s+at\s+(?:the\s+)?two\s+(?:existing\s+)?(?:injection\s+)?seams" r"|adSlots[^.]{0,60}stays?\s+in\s+the\s+template", - allow=r"not two|the head seam is not|NOT a hole", must_flag=[ "The transform emits `esi:include` markers at the two existing injection seams instead", "emit markers at the two seams", + "Emit markers at the two existing injection seams. Correction: not two; use one marker.", ], must_pass=[ "one unconditional marker at the body-close seam. Not two: the head seam is not a template hole", + "The earlier draft used two injection seams; that statement was wrong.", ], ), Check( @@ -119,9 +181,11 @@ def hits(self, normalized): ), Check( "No nonexistent Core Cache APIs", - r"(? Date: Mon, 10 Aug 2026 18:10:11 +0530 Subject: [PATCH 175/395] Remove the #1009 documentation invariant checker --- ...2026-08-08-1009-measurement-and-stage-0.md | 11 +- .../2026-08-10-1009-esi-validation-spike.md | 5 - scripts/docs-invariants.py | 292 ------------------ 3 files changed, 2 insertions(+), 306 deletions(-) delete mode 100755 scripts/docs-invariants.py diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md index d4ab8ba27..6de35c9b3 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md @@ -25,16 +25,14 @@ for wasm-safe timing, `log` for instrumentation, Viceroy for adapter tests. **Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` (§3 Steps A/B/C and §4 Stage 0). Read §4 and §5 before starting Task 5. -**Before pushing, run three gates, not two:** +**Before pushing, run both documentation gates:** ```bash cd docs && npm run format && npm run build && cd .. -python3 scripts/docs-invariants.py ``` `npm run build` is not optional — `format` passes on documents with dead links, and that -shipped a broken docs build on this branch once already. `docs-invariants.py` catches -cross-document contradictions, which neither of the other two can see. +shipped a broken docs build on this branch once already. **Two prettier gotchas, both hit while writing this plan.** CI gate 7 (`cd docs && npm run format`) runs `prettier --check` over all of `docs/`, so both bite. @@ -1042,10 +1040,5 @@ Named so nobody widens this plan mid-flight. All are specified in the spec. measuring the TTFB the publisher actually complained about; use it for before/after. - [ ] `unexpected_origin_304` rate is zero and representations are not mixed (Task 5a Step 8) — both checked **before** the win is claimed. -- [ ] **Cross-document invariants hold:** `python3 scripts/docs-invariants.py`. This is a - named gate, not a courtesy check. `npm run format` and `npm run build` catch - formatting and dead links; neither catches a claim corrected in one document and - left standing in another, which is the failure mode this document set has hit on - four separate review rounds. Add a check whenever a correction lands. - [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index 0fb273585..354a41b85 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -687,10 +687,5 @@ routes; N per arm; and the cache-tier mix. the answer. - [ ] Cleanup complete: flag resolved, C2 purged, synthetic endpoint removed, dependency landed or dropped. -- [ ] **Cross-document invariants hold:** `python3 scripts/docs-invariants.py`. This is a - named gate, not a courtesy check. `npm run format` and `npm run build` catch - formatting and dead links; neither catches a claim corrected in one document and - left standing in another, which is the failure mode this document set has hit on - four separate review rounds. Add a check whenever a correction lands. - [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/scripts/docs-invariants.py b/scripts/docs-invariants.py deleted file mode 100755 index e88fb81b2..000000000 --- a/scripts/docs-invariants.py +++ /dev/null @@ -1,292 +0,0 @@ -#!/usr/bin/env python3 -"""Cross-document invariant sweep for the #1009 document set. - -`npm run format` catches formatting. `npm run build` catches dead links. Neither -catches a claim corrected in one document and left standing in another, which is -how every review round on this branch has found real defects. - -DESIGN NOTES — a previous version of this checker reported 8/8 green on -documents that still contained the exact contradictions it claimed to check. -Three things caused that, and each is addressed here: - -1. It matched literal phrases ("two seams") that the stale text did not use - ("two existing injection seams"). Patterns are now semantic and tolerant. -2. It matched line by line, so any phrase wrapped across a line break was - invisible. Text is now whitespace-normalized per file before matching. -3. It had no way to know it had stopped working. Every check now carries - `must_flag` fixtures — strings that MUST trip it — and the script fails if - any fixture is not caught. A check that cannot fail is treated as broken. -4. It let unrelated nearby correction language excuse a violation. Generic - allow-windows are gone; the one qualified check must span the exact API - occurrence it excuses. - -The four promised input documents and both fixture directions are mandatory. -Missing inputs, unreadable inputs, or an empty fixture side make the checker -itself fail with exit 2 before it reports document results. - -A false positive costs a minute. A false green costs a merge. -""" -import re -import sys -import glob -import os - -ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") - -REQUIRED_TARGETS = frozenset( - { - "docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md", - "docs/superpowers/plans/2026-08-08-1009-measurement-findings.md", - "docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md", - "docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md", - } -) - - -def targets(): - return sorted( - glob.glob(os.path.join(ROOT, "docs/superpowers/specs/2026-08-08-esi-*.md")) - + glob.glob(os.path.join(ROOT, "docs/superpowers/plans/2026-08-*1009*.md")) - ) - - -def target_errors(paths): - """Return setup errors when any document this gate promises to scan is absent.""" - present = { - os.path.relpath(path, ROOT).replace(os.sep, "/") - for path in paths - } - return [ - f" missing required document: {path}" - for path in sorted(REQUIRED_TARGETS - present) - ] - - -def load_documents(paths): - documents = {} - errors = [] - for path in paths: - try: - with open(path, encoding="utf-8") as source: - documents[path] = normalize(source.read()) - except (OSError, UnicodeError) as error: - errors.append(f" cannot read {os.path.relpath(path, ROOT)}: {error}") - return documents, errors - - -def normalize(text): - """Collapse wraps while removing Markdown blockquote continuation markers.""" - text = re.sub(r"(?m)^\s*>\s?", "", text) - text = re.sub(r"\s*\n\s*", " ", text) - text = re.sub(r"[ \t]{2,}", " ", text) - return text - - -class Check: - def __init__(self, name, bad, must_flag=(), must_pass=()): - self.name = name - self.bad = re.compile(bad, re.I) - self.must_flag = must_flag # strings this check MUST catch - self.must_pass = must_pass # corrected strings it must NOT catch - - def hits(self, normalized): - out = [] - for m in self.bad.finditer(normalized): - out.append(normalized[max(0, m.start() - 60):m.end() + 60].strip()) - return out - - -class QualifiedOccurrenceCheck(Check): - """Flag each occurrence unless its own local clause states the required relation.""" - - def __init__(self, name, occurrence, qualified, must_flag=(), must_pass=()): - super().__init__(name, occurrence, must_flag=must_flag, must_pass=must_pass) - self.qualified = re.compile(qualified, re.I) - - def hits(self, normalized): - out = [] - qualified_spans = [match.span() for match in self.qualified.finditer(normalized)] - for m in self.bad.finditer(normalized): - if any(lo <= m.start() and m.end() <= hi for lo, hi in qualified_spans): - continue - out.append(normalized[max(0, m.start() - 60):m.end() + 60].strip()) - return out - - -CHECKS = [ - Check( - "Stage 0 gates on FINAL PASS, not a bare Vary check", - # `[*_ ]*` absorbs markdown emphasis; `gated on a curl` is its own shape - # because the sentence boundary defeats a proximity match to "Stage 0". - r"gates?[*_ ]+stage[*_ ]*0" - r"(?!\s*only\s+(?:once|after|when)\s+(?:an?\s+)?[`*_]*final\s+pass[`*_]*" - r"(?:\s+is)?\s+(?:recorded|obtained|achieved)\b)" - r"|verdict[:*_ ]+pass[*_ ]*/[*_ ]*fail" - r"|gated\s+on\s+a\s+.?curl", - must_flag=[ - "inspect the `Vary` response header. **Gates Stage 0**, the only build item", - "**Verdict:** PASS / FAIL", - "Stage 0 next, shipped as the operator flag. Gated on a `curl`, reverses an origin-load cost", - "Inspect Vary. **Gates Stage 0** immediately. A separate sentence says FINAL PASS is required.", - "Gates Stage 0 immediately; FINAL PASS gates deployment later.", - "The Vary result gates Stage 0 only on a successful curl, while FINAL PASS gates production rollout.", - ], - must_pass=[ - "Gates Stage 0 only once a FINAL PASS is recorded", - "Gates Stage 0 only after a FINAL PASS is recorded", - "**Verdict:** FINAL PASS / PROVISIONAL PASS / FAIL", - ], - ), - Check( - "Rollback is not described as config-only", - r"rolls?\s+back\s+with\s+a\s+config\s+push" - r"|rollback\s+is\s+another\s+config\s+push\s+rather", - must_flag=[ - "reverses an origin-load cost, and rolls back with a config push. Closer to a defect fix", - "This rolls back with a config push, explicitly without a purge or TTL wait.", - ], - must_pass=[ - "the read path reverts with a config push, but full rollback also needs a C1 purge path", - ], - ), - Check( - "Template has one marker, not two seams", - r"\b(?:emit|emits|use|uses|place|places|insert|inserts)\b[^.]{0,100}" - r"(?:two|both)\s+(?:existing\s+)?(?:injection\s+)?seams" - r"|markers?\s+at\s+(?:the\s+)?two\s+(?:existing\s+)?(?:injection\s+)?seams" - r"|adSlots[^.]{0,60}stays?\s+in\s+the\s+template", - must_flag=[ - "The transform emits `esi:include` markers at the two existing injection seams instead", - "emit markers at the two seams", - "Emit markers at the two existing injection seams. Correction: not two; use one marker.", - ], - must_pass=[ - "one unconditional marker at the body-close seam. Not two: the head seam is not a template hole", - "The earlier draft used two injection seams; that statement was wrong.", - ], - ), - Check( - "Headers finalize before assembly, in prose and diagrams", - r"assemble\s*(→|->)\s*finaliz" - r"|finaliz\w*[^.]{0,40}(runs?|still run)[^.]{0,20}after\s+assembly", - must_flag=[ - "`origin → lol_html transform → fastly::cache::core → assemble → finalize`", - "esi assemble → finalize → client", - "Cookie and privacy finalization still run after assembly", - ], - must_pass=[ - "fastly::cache::core → finalize headers → stream assembly", - "Cookie and privacy finalization ran BEFORE assembly, not after", - ], - ), - Check( - "No nonexistent Core Cache APIs", - r"\b[A-Za-z_]\w*\.to_body\s*\(\s*\)", - must_flag=[ - "let template = found.to_body();", - "let template = found.to_body(); // there is no fallback", - ], - must_pass=["found.to_stream()? // fallible; there is no `to_body()`"], - ), - Check( - "A2/A3 not decided on root TTFB", - r"A3\s+beats\s+A2\s+on\s+TTFB", - must_flag=["2. A3 beats A2 on TTFB by a margin the reviewers ratify"], - must_pass=["A3 beats A2 on bids-ready time, adInit fire time, and first attributed paint"], - ), - Check( - "ESI not described as impossible", - r"\bESI\s+(?:is|remains)\s+(?:not\s+viable|structurally\s+(?:blocked|impossible))", - must_flag=[ - "Answers #1009: ESI is not viable here, for a structural reason", - "ESI is not viable here. An earlier revision discussed a different problem.", - ], - must_pass=["The first revision concluded that ESI was structurally blocked. Both claims are false"], - ), - QualifiedOccurrenceCheck( - "C1 and C2 purge surfaces not conflated", - r"InsertBuilder::surrogate_keys", - r"InsertBuilder::surrogate_keys(?:\(\[\.\.\.\]\))?" - r"(?:(?!InsertBuilder::surrogate_keys|[.]|\bnot\b|\bnever\b|\bno\s+longer\b).){0,60}" - r"(?:\(\s*Core\s+Cache\s*\)|(?:is|belongs\s+to)" - r"(?![^.]{0,20}\b(?:not|never)\b|[^.]{0,20}\bno\s+longer\b)" - r"[^.]{0,60}Core\s+Cache)", - must_flag=[ - "Purge is available in-process, with keys attached at insert via `InsertBuilder::surrogate_keys`. Rollback is therefore flip then purge.", - "Attach C1 keys via `InsertBuilder::surrogate_keys`. C2 is described below.", - "C1 uses `InsertBuilder::surrogate_keys`. Separately, `InsertBuilder::surrogate_keys` is the Core Cache API for C2.", - "C1 uses `InsertBuilder::surrogate_keys`, but `InsertBuilder::surrogate_keys` is the Core Cache API for C2.", - "`InsertBuilder::surrogate_keys` is the Core Cache API for C2, but C1 uses `InsertBuilder::surrogate_keys`.", - "C1 uses `InsertBuilder::surrogate_keys`; it is not the Core Cache API.", - "C1 uses `InsertBuilder::surrogate_keys`; it no longer belongs to the Core Cache API.", - "This is not C2; `InsertBuilder::surrogate_keys` powers C1.", - ], - must_pass=[ - "`InsertBuilder::surrogate_keys` is the Core Cache API and keys C2. It does **not** key C1.", - ], - ), -] - - -def self_test(): - """A check that cannot fail is broken. Prove each one still fires.""" - broken = [] - for c in CHECKS: - if not c.must_flag: - broken.append(f" {c.name!r}: has no must_flag fixtures") - if not c.must_pass: - broken.append(f" {c.name!r}: has no must_pass fixtures") - for bad in c.must_flag: - if not c.hits(normalize(bad)): - broken.append(f" {c.name!r}: FAILED to flag known-bad text:\n {bad[:110]}") - for good in c.must_pass: - if c.hits(normalize(good)): - broken.append(f" {c.name!r}: wrongly flagged corrected text:\n {good[:110]}") - - if not target_errors([]): - broken.append(" target guard accepted an empty document set") - synthetic_targets = [os.path.join(ROOT, path) for path in REQUIRED_TARGETS] - if errors := target_errors(synthetic_targets): - broken.append(f" target guard rejected its required manifest: {errors}") - return broken - - -def report_checker_errors(errors): - print("CHECKER IS BROKEN — its own fixtures or document setup do not pass.") - print("A green run from this state would be meaningless.\n") - print("\n".join(errors)) - - -def main(): - paths = targets() - broken = self_test() + target_errors(paths) - if broken: - report_checker_errors(broken) - return 2 - - documents, read_errors = load_documents(paths) - if read_errors: - report_checker_errors(read_errors) - return 2 - - fail = False - for c in CHECKS: - found = [] - for path, document in documents.items(): - for h in c.hits(document): - found.append(f" {os.path.relpath(path, ROOT)}: …{h}…") - if found: - fail = True - print(f"\n[FAIL] {c.name}") - print("\n".join(found)) - else: - print(f"[ok] {c.name}") - - print() - print("Contradictions found — do not push." if fail else - f"All {len(CHECKS)} invariants hold (self-test passed).") - return 1 if fail else 0 - - -if __name__ == "__main__": - sys.exit(main()) From 62c4b703e754e0a955cb383bbfdb4f84060e54e1 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 18:27:40 +0530 Subject: [PATCH 176/395] Align the #1009 spec structure with what it actually covers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three structural fixes, no content changes. The title said "ESI and the Cacheable Root" while the actionable front of the document — sections 1 through 4 — is entirely Stage 0. ESI now lives in one section, one appendix, and mostly in a separate plan. Retitled to match. The filename keeps its esi- prefix deliberately: the commit history and every cross-reference point at it, and renaming would cost more than the mismatch. Added a document map. Three documents answer #1009 and nothing said which owns what, which is the seam every cross-document contradiction has appeared in. It also tells a reader arriving from the issue where the ESI answer actually is, rather than leaving them to infer it from a Stage 0 design document. Consolidated the staging. Stage 0 lived in section 4 while Stages 1 through 5 lived in section 7, so the sequence was split across two places, and Stage 5 had become an entry that read "superseded, see the other plan" — a staging list containing something that is not a stage. There is now one table, Stage 5 is gone, and ESI is stated as running independently of Stages 1 through 4 rather than queued behind them. Two stale "Stages 3b-5" ranges followed from that and are corrected. --- ...08-esi-cacheable-root-validation-design.md | 49 +++++++++++++++---- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md index 39a65dac5..40b2b68f1 100644 --- a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -1,4 +1,7 @@ -# ESI and the Cacheable Root +# The Cacheable Root: Latency Diagnosis and Stage 0 Design + +_Filename retains its original `esi-` prefix; the commit history and every +cross-reference point at it. The subject moved, the path did not._ **Issue:** IABTechLab/trusted-server#1009 · **Date:** 2026-08-08 · **Revised:** 2026-08-10 @@ -36,6 +39,22 @@ > here is the latency re-diagnosis and the Stage 0 optimisation, which are worth doing > and are **not** an answer to #1009. +## Document map — read this first + +#1009 is answered across three documents, not one. This is the only place that says +which owns what. + +| Document | Owns | +| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------- | +| **This spec** | Why the TTFB regression happens, what Stage 0 is and why, and the corrected ESI feasibility verdict | +| [Stage 0 plan](../plans/2026-08-08-1009-measurement-and-stage-0.md) | Implementing the measurement and the cache-bypass flag. **Does not close #1009.** | +| [ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md) | **Where #1009 is actually decided.** Four arms, safety gates, decision rule. | +| [Findings](../plans/2026-08-08-1009-measurement-findings.md) | Recorded results. Currently: Step A only, at `PROVISIONAL PASS`. | + +**If you want the ESI answer**, it is [§2](#2-why--the-three-findings) for the verdict, +[§6.6](#66-the-esi-pipeline-corrected) for the pipeline, and the spike plan for how it +gets validated. Everything else here is Stage 0 and the latency analysis behind it. + **Decision requested:** approve the four items in §1. > **Orientation.** #1009 asks whether Edge Side Includes (ESI) can cache page fragments @@ -67,7 +86,7 @@ | D1 | **ESI is feasible and unvalidated.** Validate it via [the spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), not by deferring it. | Eng + product | | D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | | D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to a **`FINAL PASS`** in §3. A `Vary` check alone is a `PROVISIONAL PASS` and is not a release gate. Rollback needs a purge path, not only a config push. | Eng | -| D4 | **Stages 1–2 queue behind the SSAT price defect and #938.** Stages 3b–5 unscheduled. | Product | +| D4 | **Stages 1–2 queue behind the SSAT price defect and #938.** Stages 3b–4 unscheduled. ESI is not in this queue — see §7. | Product | Rationale for D4 in [§8](#8-priority). Everything this document recommends _against_ doing is in [§7](#7-deferred-work-specified-not-scheduled), at deliberately lower @@ -501,7 +520,25 @@ This applies to any shared-template work, ESI or client-fill alike. The ## 7. Deferred work, specified not scheduled -Lower detail is deliberate. Full specifications are in the appendices. +**The full sequence, in one place.** Stage 0 is specified in [§4](#4-stage-0--the-only-build-item-recommended-now) +rather than repeated here; everything below it is deferred. + +| Stage | What | Status | +| ----- | ----------------------------------------------- | ---------------------------------------------- | +| **0** | Operator flag disabling the origin cache bypass | Recommended now. Gated on a `FINAL PASS`. §4. | +| 1 | Bid delivery off the response body | Deferred behind the correctness defects | +| 2 | Delete the `` hold | Deferred; one-way, needs a Stage 1 soak | +| 3a | Browser caching (`private, max-age` + `ETag`) | Specified, low risk, unscheduled | +| 3b | Shared cacheability | Blocked on geo suppression, `Vary`, and Step B | +| 4 | Purge wiring | Prerequisite for any TS-owned cache | + +**ESI is not a stage here.** It was Stage 5 in an earlier revision, queued behind the +rest. It no longer queues: it is feasible on the pinned SDK and is decided by +[its own spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), which runs +independently of Stages 1–4. The shared template cache it needs is `fastly::cache::core` +([§6.6](#66-the-esi-pipeline-corrected)), not a new service. + +Lower detail below is deliberate. Full specifications are in the appendices. **Stage 1 — bid delivery off the response body.** The client fetches `/_ts/page-bids` at navigation generation 0. Endpoint, same-origin gate, wire shape, and client consumer @@ -558,12 +595,6 @@ cache — not the HTTP read-through cache C1. This is **missing wiring, not a platform limit.** Until it exists, any TS-owned cache is TTL-only and a config push takes up to one TTL to take effect. -**Stage 5 — ESI.** Superseded. ESI no longer waits on a "revival condition"; it is -feasible on the pinned SDK and is validated by -[its own spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), which does not -queue behind Stages 1–4. The shared template cache it needs is -`fastly::cache::core` ([§6.6](#66-the-esi-pipeline-corrected)), not a new service. - **Identity needs no work.** A new visitor's first navigation sets the EC cookie and the privacy net downgrades that one response; every later navigation sets no cookie and is cacheable. First-visit parity is a non-goal; if ever wanted, move cookie issuance onto From bcf2fd26d7c3209d325a0100d55510f268ed944a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 18:57:55 +0530 Subject: [PATCH 177/395] Add the esi crate to the Fastly adapter and record Task 1 as passed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cheapest falsifier for #1009 clears. esi 0.7.1 compiles clean on Rust 1.95.0 for wasm32-wasip1, all six clippy targets pass, format is clean, and the integration-tests crate still resolves. ESI is not blocked by this toolchain. Nine new transitive dependencies, none of them displacing an existing one: esi, nom 8, rand 0.10, rand_core 0.10, chacha20, cpufeatures, atoi, html-escape, md5. regex stays at 1.12.4, bytes at 1.12.0 and log at 0.4.33. nom and rand gain new majors that coexist with the versions already in the tree rather than replacing them, which is the outcome that keeps this cheap — a forced bump on a shared dependency is what would have made it expensive. The dependency is added and unused. It belongs to the Fastly adapter rather than trusted-server-core, because the crate is hard-bound to fastly::{Request, Response, Backend} and core has to stay portable across the four adapters. Also corrects a claim in the spike plan that this task falsified. Step 3 told the implementer to check for a desync between the root lockfile and one at crates/trusted-server-integration-tests/Cargo.lock. That file does not exist: the crate is a workspace member and shares the root lockfile, so the hazard cannot arise in that form. The step now checks the thing that does matter, which is whether an existing shared dependency was forced to move. Compiling is not working. Nothing here exercises cache::core, ESI assembly, or any runtime behaviour, and Tasks 2 onward are untouched. --- Cargo.lock | 112 ++++++++++++++++-- .../trusted-server-adapter-fastly/Cargo.toml | 1 + .../2026-08-08-1009-measurement-findings.md | 37 ++++++ .../2026-08-10-1009-esi-validation-spike.md | 28 +++-- 4 files changed, 160 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb8f40c68..b21216fa3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,7 +146,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom", + "nom 7.1.3", "num-traits", "rusticata-macros", "thiserror 1.0.69", @@ -254,6 +254,15 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -573,7 +582,18 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", ] [[package]] @@ -583,7 +603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -916,6 +936,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1041,7 +1070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", @@ -1186,7 +1215,7 @@ checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" dependencies = [ "asn1-rs", "displaydoc", - "nom", + "nom 7.1.3", "num-bigint", "num-traits", "rusticata-macros", @@ -1690,6 +1719,27 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "esi" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e384a711090b57e3dd20080915935607078ab0b43d49575994b44dd36956f84" +dependencies = [ + "atoi", + "base64", + "bytes", + "chrono", + "fastly", + "html-escape", + "log", + "md5", + "nom 8.0.0", + "percent-encoding", + "rand 0.10.2", + "regex", + "thiserror 2.0.18", +] + [[package]] name = "etcetera" version = "0.10.0" @@ -2034,6 +2084,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -2188,6 +2239,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "html-escape" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9356095b4b41197bba32173600e1582792cda618f65d12f68e2e77d273413c5" + [[package]] name = "html5ever" version = "0.35.0" @@ -2914,6 +2971,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" +[[package]] +name = "md5" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" + [[package]] name = "memchr" version = "2.8.2" @@ -2984,6 +3047,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "num" version = "0.4.3" @@ -3477,7 +3549,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -3775,6 +3847,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20 0.10.1", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3813,6 +3896,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rcgen" version = "0.13.2" @@ -4079,7 +4168,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -4494,7 +4583,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -4506,7 +4595,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.9.0", "opaque-debug", ] @@ -4518,7 +4607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -5276,6 +5365,7 @@ dependencies = [ "edgezero-adapter-fastly", "edgezero-core", "error-stack", + "esi", "fastly", "fern", "futures", @@ -6325,7 +6415,7 @@ dependencies = [ "data-encoding", "der-parser", "lazy_static", - "nom", + "nom 7.1.3", "oid-registry", "ring", "rusticata-macros", diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index b6bc0f1a1..cf73a2040 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -18,6 +18,7 @@ chrono = { workspace = true } edgezero-adapter-fastly = { workspace = true, features = ["fastly"] } edgezero-core = { workspace = true } error-stack = { workspace = true } +esi = "0.7" fastly = { workspace = true } fern = { workspace = true } futures = { workspace = true } diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md index a092f6c52..831101f7b 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -152,6 +152,43 @@ available (or roll a versioned key namespace), **then** observe past the origin declaring the incident closed. With no C1 purge path wired, the tail is the TTL itself — roughly a minute here, and a recorded risk rather than a surprise. +## ESI spike Task 1 — does `esi` 0.7 build on this toolchain? + +**Date:** 2026-08-10 · **Verdict: PASS.** The cheapest falsifier for the ESI question +clears. #1009 is not closed by a toolchain limit. + +| Check | Result | +| ------------------------------------------------------------------------------------------ | -------------------- | +| `cargo add esi@0.7 --package trusted-server-adapter-fastly` | resolved `esi 0.7.1` | +| `cargo check-fastly` (Rust 1.95.0 / `wasm32-wasip1`) | clean | +| `cargo fmt --all -- --check` | clean | +| All six clippy targets (fastly, axum, cloudflare, cloudflare-wasm, spin-native, spin-wasm) | clean | +| `cargo check --manifest-path crates/trusted-server-integration-tests/Cargo.toml --tests` | clean | + +**Nine new transitive dependencies:** `esi 0.7.1`, `nom 8.0.0`, `rand 0.10.2`, +`rand_core 0.10.1`, `chacha20 0.10.1`, `cpufeatures 0.3.0`, `atoi 2.0.0`, +`html-escape 0.2.15`, `md5 0.8.1`. + +**No existing shared dependency moved.** `regex` stays 1.12.4, `bytes` 1.12.0, `log` +0.4.33. `nom` and `rand` gain new majors that coexist with the existing 7.1.3 / 0.8.6 / +0.9.4 rather than replacing them — the best available outcome, since a forced bump on a +shared dep is what would have made this expensive. + +### A claim in the spike plan was wrong + +Task 1 Step 3 told the implementer to check for a desync between the root `Cargo.lock` and +`crates/trusted-server-integration-tests/Cargo.lock`. **That second lockfile does not +exist.** The integration-tests crate is a workspace member (root `Cargo.toml:10`) and +shares the root lockfile, so the desync hazard cannot arise in that form. The plan has +been corrected. The dual-lockfile constraint was real at some earlier point; it is not the +current layout. + +### Not yet verified + +Compiling is not working. Nothing here exercises `cache::core`, ESI assembly, or any +runtime behaviour — Tasks 2 onward remain untouched, and the `esi` dependency is added but +unused. + ## Step B — consumers of TS's own response headers Not yet run. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index 354a41b85..4bc3cc3b5 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -137,21 +137,35 @@ cargo check-fastly Expected: clean. The crate declares edition 2021 with no `rust-version`, and pulls recent `rand` and `nom`, so this is a genuine question on Rust 1.95.0 / `wasm32-wasip1`. -- [ ] **Step 3: Check the lockfiles have not desynced** +- [ ] **Step 3: Check no shared dependency was forced to move** ```bash git diff --stat Cargo.lock -cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +cargo check --manifest-path crates/trusted-server-integration-tests/Cargo.toml --tests \ + --target "$(rustc -vV | sed -n 's/^host: //p')" ``` -CI requires shared direct deps to match between the root and integration-tests lockfiles. -`regex`, `bytes`, and `log` overlap. If they desync, fix with targeted -`cargo update -p --precise ` — **never a full update**. +**Correction, verified 2026-08-10:** an earlier revision of this step warned about a +desync between the root `Cargo.lock` and `crates/trusted-server-integration-tests/Cargo.lock`. +**That second lockfile does not exist** — the crate is a workspace member (root +`Cargo.toml:10`) and shares the root lockfile. The hazard cannot arise in that form. + +What does matter is whether adding `esi` forces an **existing** shared dependency to a new +version, since `regex`, `bytes`, and `log` are used across the workspace. Adding a new +major that coexists is harmless; moving an existing one is not. If one moves, fix with a +targeted `cargo update -p --precise ` — **never a full update**. + +**Already run and recorded** in [the findings](./2026-08-08-1009-measurement-findings.md): +no existing shared dependency moved. - [ ] **Step 4: Record and commit, or stop** -If Step 2 fails, this plan stops here and #1009 is answered "not on this toolchain." -Record that in the findings document and escalate rather than fighting the build. +**Task 1 is complete — verdict PASS, recorded 2026-08-10.** `esi` 0.7.1 compiles clean on +Rust 1.95.0 / `wasm32-wasip1`, all six clippy targets pass, and no existing shared +dependency moved. See [the findings](./2026-08-08-1009-measurement-findings.md). + +Had Step 2 failed, this plan would have stopped here with #1009 answered "not on this +toolchain." It did not. ```bash git add crates/trusted-server-adapter-fastly/Cargo.toml Cargo.lock From b35f8df4d39b99183480f54c54094d3ea88e072a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 19:05:18 +0530 Subject: [PATCH 178/395] Re-sequence the ESI spike for local-first validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit You asked whether a Fastly test service is really needed. Probed it rather than reasoned about it: Viceroy 0.17 implements the whole Core Cache surface this spike uses. A temporary test under cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1 exercised insert/finish/lookup/to_stream, and then the shape Task 3 Step 4 actually specifies — Transaction::lookup, must_insert_or_update, insert(...).surrogate_keys(...).execute_and_stream_back(), and hit-after-insert. All passed. The probe is removed; the result is recorded in the findings. So provisioning is not a prerequisite. An earlier revision made it Task 2 and a blocker on everything downstream, which would have stalled the spike on infrastructure it does not need yet. Almost all the correctness and safety work runs locally: the C2 cache logic, the transform, template byte-identity, ESI assembly (the crate is pure Rust over BufRead/Write), DCA and dispatcher refusal, fragment-failure degradation, header ordering, and the leakage gates. Task 2 is now scoped to what genuinely needs a real service and is no longer on the critical path; the dependency graph reflects that. Two caveats recorded rather than glossed. Viceroy is a single instance, so a passing Transaction test proves the API works and not that request collapsing behaves under load. And local timings are meaningless for Task 7's decision rule — every performance number still needs the real service. --- .../2026-08-08-1009-measurement-findings.md | 34 ++++++++- .../2026-08-10-1009-esi-validation-spike.md | 76 +++++++++++++------ 2 files changed, 83 insertions(+), 27 deletions(-) diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md index 831101f7b..9e68acb27 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -183,11 +183,39 @@ shares the root lockfile, so the desync hazard cannot arise in that form. The pl been corrected. The dual-lockfile constraint was real at some earlier point; it is not the current layout. +### Viceroy 0.17 supports the whole Core Cache surface this spike needs + +**Date:** 2026-08-10 · **Verdict: PASS.** Probed directly under +`cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1`, then removed: + +| API | Result | +| ------------------------------------------------------------------------ | ------ | +| `cache::core::insert(key, ttl).execute()` → write → `finish()` | works | +| `cache::core::lookup(key).execute()` → `Found::to_stream()` | works | +| `Transaction::lookup(key).execute()` → `must_insert_or_update()` | works | +| `Transaction::insert(ttl).surrogate_keys([…]).execute_and_stream_back()` | works | +| Second transactional lookup reports a hit, no obligation | works | + +That is the entire API surface the spike's Task 3 Step 4 specifies, including the +transaction and stream-back shapes. + +**Consequence: provisioning a Fastly service is not a prerequisite.** An earlier revision +of the spike plan made it Task 2 and a blocker on everything downstream. Almost all of the +correctness and safety work — the C2 cache logic, the transform, template byte-identity, +ESI assembly, DCA and dispatcher refusal, fragment-failure degradation, header ordering, +and the leakage gates — runs locally. The plan is re-sequenced accordingly. + +**What still needs a real service:** shielding behaviour, POP-level cache tiering, +request collapsing under genuine concurrency (Viceroy is a single instance, so a passing +`Transaction` test proves the API works and not that collapsing is correct under load), +stale revalidation timing, and **every performance number in the decision rule**. Local +timings are meaningless for the decision. + ### Not yet verified -Compiling is not working. Nothing here exercises `cache::core`, ESI assembly, or any -runtime behaviour — Tasks 2 onward remain untouched, and the `esi` dependency is added but -unused. +Compiling and a cache round-trip are not an implementation. Nothing yet exercises the +`lol_html` transform into C2, ESI assembly, or any runtime behaviour on the publisher path, +and the `esi` dependency is added but unused. ## Step B — consumers of TS's own response headers diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index 4bc3cc3b5..f1f3776f5 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -97,19 +97,19 @@ not its implementation. ## Task order and dependencies ``` -Stage 0 plan (flag + timing instrumentation) ──┐ - ├──> Task 3 (C2 template cache) -Task 1 (esi crate compiles) ───────────────────┤ -Task 2 (test service + harness) ────────────────┘ │ - ├──> Task 4 (A2 client-fill) - ├──> Task 5 (A3 ESI) - └──> Task 6 (safety gates) - │ - └──> Task 7 (decision record) +Task 1 (esi compiles) ── DONE, PASS ──┐ + ├──> Task 3 (C2 cache) ─┬──> Task 4 (A2 client-fill) +Stage 0 plan (flag + instrumentation) ┘ ├──> Task 5 (A3 ESI) + └──> Task 6 (safety gates) + │ + Task 2 (real service) ─────────────────────────────┴──> Task 7 (decision) ``` -Tasks 1 and 2 are independent and should run first — both can invalidate the plan -cheaply. Task 6 runs against every arm, not once at the end. +**Task 2 is not a blocker on Tasks 3–6.** Everything those tasks need is exercisable under +Viceroy 0.17 — verified, see Task 2. The real service is required only for the +measurements Task 7 decides on, so provision it once there is something worth measuring. + +Task 6 runs against every arm, not once at the end. --- @@ -174,19 +174,47 @@ git commit -m "Add the esi crate to the Fastly adapter for the #1009 validation --- -## Task 2: Stand up the test service and the harness - -**Viceroy 0.17 does support `cache::core` locally** — an earlier draft of this plan said -otherwise and was wrong. What it does **not** support is the customized HTTP -read-through hooks (`after_send` / `set_body_transform`), which matters only if the -alternative design in Task 3 Step 4 is chosen. - -So: C2 insert/lookup/transaction logic, the transform, and the security properties are all -testable locally. **Shielding, request collapsing under real concurrency, POP behaviour, -and stale revalidation are not** — those need a real Fastly service. Establish one before -Tasks 3–6, and be clear which findings came from which environment. - -- [ ] **Step 1: Provision a dedicated test service** +## Task 2: Local validation first, real service only for what needs it + +**Verified 2026-08-10 under Viceroy 0.17: the entire Core Cache surface this spike uses +works locally.** A probe exercised `cache::core::insert`, `lookup`, `finish`, `to_stream`, +and — the shape Task 3 Step 4 actually specifies — `Transaction::lookup`, +`must_insert_or_update`, `insert(...).surrogate_keys(...).execute_and_stream_back()`, and +hit-after-insert semantics. All passed. Recorded in +[the findings](./2026-08-08-1009-measurement-findings.md). + +That reorders this plan. An earlier revision made provisioning a Fastly service Task 2 and +a blocker on everything after it. It is not a blocker: **almost all of the correctness and +safety work is local**, and only the numbers and the cache topology need real +infrastructure. + +| Work | Where | +| ------------------------------------------------------------ | ------------ | +| C2 insert / lookup / transaction logic (Task 3) | **Local** | +| The `lol_html` transform and template byte-identity (Task 3) | **Local** | +| ESI assembly — the crate is pure Rust over `BufRead`/`Write` | **Local** | +| DCA off, dispatcher allowlist, injection refusal (Task 5) | **Local** | +| Fragment-failure degradation (Task 5) | **Local** | +| Header-finalization ordering, no-C3 assertions (Task 6) | **Local** | +| Cross-user leakage / request-neutrality gates (Task 6) | **Local** | +| Shielding behaviour | Real service | +| POP-level cache tiering (`x-cache`, `hit-state`, `age`) | Real service | +| Request collapsing under genuine concurrency | Real service | +| Stale revalidation timing at the edge | Real service | +| **Every performance number in Task 7's decision rule** | Real service | + +**So: build and prove correctness locally through Tasks 3, 5, and 6 before provisioning +anything.** If the design is wrong or leaks, that surfaces locally for free, and the +service is only needed once there is something worth measuring. + +Two caveats on the local scope. Viceroy is a single instance, so a passing `Transaction` +test proves the API works, **not** that collapsing behaves correctly under load. And local +timings are meaningless for the decision — do not let a fast local run substitute for +Task 7 evidence. + +### When the real service is needed + +- [ ] **Step 1: Provision it — after local correctness passes, not before** Separate from production. Confirm and record: whether the publisher backend is **shielded**, and whether any Delivery service fronts the Compute service. Both change From 7781009d4b5de15b2a2d5b2bc5292a44e0feebc1 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 19:27:30 +0530 Subject: [PATCH 179/395] Add AssemblyMode and gate the head seam on template neutrality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First implementation step of the #1009 ESI spike. No behaviour change: the mode defaults to Inline and every existing path is unaffected. AssemblyMode lives on CreativeOpportunitiesConfig as Option with skip_serializing_if, following the section_root pattern already established there. The reason is in that struct's own doc comments: these types use deny_unknown_fields, so a pushed key makes an older binary fail configuration load. Keeping the key absent when unset means a deployment that never sets it stays rollback-compatible. A test asserts the unset value is not serialized, so that property cannot regress silently. The head seam now goes through template_ad_slots_script rather than an inline conditional. Under Inline it keeps today's behaviour, emitting adSlots only when the ad stack runs, which is correct for a response that is never shared. Under ClientFill and Esi it returns None unconditionally, because should_run_ad_stack folds in consent, bot classification, prefetch status and the auction kill switch. A shared template that emitted conditionally would freeze the first-filling request's decision for every later reader: a consent-denied fill would serve a no-ads template to consenting users, and a consenting fill would serve ad markup to someone who refused. Three tests, and the shape of them matters. An absence-of-per-user-values scan would have passed the broken design, because adSlots content really is derived from config and path. What catches it is byte-identity across requests differing only in the gating decision, so that is what is asserted — including across differing slot matches. The inline test exists so a future change cannot make the shared-mode assertions pass by breaking the shipped path. Extracting the decision as a pure function is deliberate: it makes the invariant testable without driving the whole pipeline, which is what let these tests be written before any cache work exists. Verified: fmt, all six clippy targets, and all four adapter suites, including 1838 core tests under Viceroy. --- .../src/creative_opportunities.rs | 113 +++++++++++ crates/trusted-server-core/src/publisher.rs | 176 +++++++++++++++++- 2 files changed, 281 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 10a85b3e8..94fd2fce1 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -183,6 +183,36 @@ fn derive_section(path: &str, section_root: &str, section_segment: usize) -> Str } } +/// How per-user ad state reaches the page. +/// +/// `Inline` is the shipped behaviour: the auction result is injected before +/// `` and the root document is therefore uncacheable. The other two serve +/// a request-neutral shared template and fill the per-user holes afterwards — +/// `ClientFill` from the browser, `Esi` at the edge. +/// +/// Spike-only, for the #1009 ESI validation. Remove with the spike. +/// +/// # Why the template must be request-neutral +/// +/// Under `ClientFill` and `Esi` the template is shared across visitors, so +/// nothing whose *presence* depends on the request may appear in it — not merely +/// nothing whose *value* does. `tsjs.adSlots` is the trap: its content is derived +/// from config and path, but whether it is emitted at all is gated on consent, +/// bot classification, prefetch status and the auction kill switch. A template +/// filled by the first request would freeze that request's decision for every +/// later reader. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AssemblyMode { + /// Inject bids inline before ``. Root uncacheable. Shipped behaviour. + #[default] + Inline, + /// Serve a shared template; the browser fetches the per-user fragment. + ClientFill, + /// Serve a shared template; assemble the fragment at the edge with ESI. + Esi, +} + /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -244,11 +274,30 @@ pub struct CreativeOpportunitiesConfig { /// [`section_root`](Self::section_root) are omitted. #[serde(default, skip_serializing_if = "Option::is_none")] pub section_segment: Option, + /// How per-user ad state reaches the page. Absent means + /// [`AssemblyMode::Inline`], the shipped behaviour. + /// + /// `Option` rather than a bare enum, and `skip_serializing_if`, deliberately: + /// these structs use `deny_unknown_fields`, so a pushed key makes an older + /// binary fail configuration load. Keeping it absent when unset means a + /// deployment that never sets it stays rollback-compatible. + /// + /// Spike-only. See [`AssemblyMode`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub assembly_mode: Option, /// Slot templates. Empty vec = feature disabled (no auction fired, no globals injected). #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub slot: Vec, } +impl CreativeOpportunitiesConfig { + /// Resolved assembly mode, defaulting to [`AssemblyMode::Inline`] when unset. + #[must_use] + pub fn assembly_mode(&self) -> AssemblyMode { + self.assembly_mode.unwrap_or_default() + } +} + impl CreativeOpportunitiesConfig { /// Derives the `{section}` value for `path` under this config's section /// policy ([`section_root`](Self::section_root) and @@ -1149,6 +1198,7 @@ mod tests { auction_timeout_ms: None, price_granularity: PriceGranularity::default(), section_root: section_root.map(str::to_string), + assembly_mode: None, section_segment: None, slot: vec![slot], } @@ -1546,6 +1596,7 @@ mod tests { auction_timeout_ms: None, price_granularity: PriceGranularity::default(), section_root: None, + assembly_mode: None, section_segment: None, slot: Vec::new(), }; @@ -1824,6 +1875,68 @@ mod tests { ); } + #[test] + fn assembly_mode_defaults_to_inline_when_absent() { + // Arrange: the minimal config an existing deployment would have. + let toml = r#" + gam_network_id = "99999" + "#; + + // Act + let config: CreativeOpportunitiesConfig = + toml::from_str(toml).expect("should deserialize without assembly_mode"); + + // Assert + assert_eq!( + config.assembly_mode, None, + "an absent key should stay absent rather than materializing a value" + ); + assert_eq!( + config.assembly_mode(), + AssemblyMode::Inline, + "should resolve to the shipped inline behaviour" + ); + } + + #[test] + fn assembly_mode_deserializes_each_variant() { + for (raw, expected) in [ + ("inline", AssemblyMode::Inline), + ("client_fill", AssemblyMode::ClientFill), + ("esi", AssemblyMode::Esi), + ] { + let toml = format!( + r#" + gam_network_id = "99999" + assembly_mode = "{raw}" + "# + ); + let config: CreativeOpportunitiesConfig = + toml::from_str(&toml).unwrap_or_else(|e| panic!("should parse {raw}: {e}")); + assert_eq!( + config.assembly_mode(), + expected, + "should resolve `{raw}` to {expected:?}" + ); + } + } + + #[test] + fn unset_assembly_mode_is_omitted_from_serialized_config() { + // `deny_unknown_fields` means a pushed key breaks config load on an older + // binary. A deployment that never sets this must not gain the key just by + // round-tripping through a newer one. + let config: CreativeOpportunitiesConfig = + toml::from_str("gam_network_id = \"99999\"").expect("should deserialize"); + + let serialized = toml::to_string(&config).expect("should serialize"); + + assert!( + !serialized.contains("assembly_mode"), + "unset assembly_mode must not be serialized, got:\n{serialized}" + ); + } + #[test] fn prebid_slot_params_deserializes_without_bidders_field() { let json = r#"{"bidders": {}}"#; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index efb9dd4f6..09d033d76 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -50,6 +50,7 @@ use crate::auction::types::{ use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; +use crate::creative_opportunities::{AssemblyMode, CreativeOpportunitiesConfig}; use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; @@ -2917,14 +2918,18 @@ pub async fn handle_publisher_request( crate::integrations::gpt_diagnostics::finalize_response(&gpt_diagnostics, &mut response); - let ad_slots_script = if should_run_ad_stack { - settings - .creative_opportunities - .as_ref() - .map(|co_config| build_ad_slots_script(&matched_slots, co_config, &request_path)) - } else { - None - }; + let assembly_mode = settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::assembly_mode) + .unwrap_or_default(); + let ad_slots_script = template_ad_slots_script( + assembly_mode, + should_run_ad_stack, + settings, + &matched_slots, + &request_path, + ); // §4.7: HTML with synthesized per-navigation auction state must not be // stored or validated as an origin representation. Strip both browser and @@ -3555,6 +3560,45 @@ fn match_renderable_slots( /// /// Property names match what the client-side TSJS bundle expects: /// `gam_unit_path`, `div_id`, `formats`, and `targeting`. +/// What the `` seam injects, given the assembly mode. +/// +/// Under [`AssemblyMode::Inline`] the response is per-navigation and not shared, +/// so emitting `tsjs.adSlots` only when the ad stack runs is correct. +/// +/// Under [`AssemblyMode::ClientFill`] and [`AssemblyMode::Esi`] the document is a +/// **shared template**, and `should_run_ad_stack` is request-dependent — it folds +/// in consent, bot classification, prefetch status and the auction kill switch. +/// Emitting conditionally there would freeze the first-filling request's decision +/// for every later reader of the cached object: a consent-denied fill would serve +/// a no-ads template to consenting users, and a consenting fill would serve ad +/// markup to someone who refused. +/// +/// So those modes return [`None`] **unconditionally**, and `adSlots` moves to the +/// per-request fragment alongside the bids. The head seam is not a template hole. +/// +/// See `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +/// §6.7. +pub(crate) fn template_ad_slots_script( + mode: AssemblyMode, + should_run_ad_stack: bool, + settings: &Settings, + matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + request_path: &str, +) -> Option { + match mode { + AssemblyMode::ClientFill | AssemblyMode::Esi => None, + AssemblyMode::Inline => { + if !should_run_ad_stack { + return None; + } + settings + .creative_opportunities + .as_ref() + .map(|co_config| build_ad_slots_script(matched_slots, co_config, request_path)) + } + } +} + pub(crate) fn build_ad_slots_script( matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, @@ -4538,6 +4582,121 @@ mod tests { .expect("should proxy publisher request") } + mod template_neutrality_tests { + //! The gate for #1009's shared-template design. + //! + //! An "absence of per-user values" scan is not sufficient here: the bug + //! that nearly shipped was a *conditionally present* element whose own + //! content was per-URL. These tests assert byte-identity across requests + //! that differ only in the gating decision. + + use super::*; + use crate::creative_opportunities::{ + AssemblyMode, CreativeOpportunityFormat, CreativeOpportunitySlot, + }; + + fn slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: "atf".to_string(), + gam_unit_path: Some("/99999/example/home".to_string()), + div_id: Some("ad-atf".to_string()), + page_patterns: vec!["/**".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }], + floor_price: None, + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + compiled_unit: None, + } + } + + fn settings_with_slots() -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + // Construct the section rather than mutating it if present: the shared + // fixture does not carry `[creative_opportunities]`, and an `if let + // Some(..)` here would silently no-op and make the inline assertion + // below vacuous. + settings.creative_opportunities = Some(CreativeOpportunitiesConfig { + gam_network_id: "99999".to_string(), + auction_timeout_ms: Some(500), + price_granularity: Default::default(), + section_root: None, + assembly_mode: None, + section_segment: None, + slot: vec![slot()], + }); + settings + } + + #[test] + fn shared_modes_emit_no_head_script_regardless_of_the_gating_decision() { + let settings = settings_with_slots(); + let slots = [slot()]; + + for mode in [AssemblyMode::ClientFill, AssemblyMode::Esi] { + let ran = template_ad_slots_script(mode, true, &settings, &slots, "/"); + let did_not_run = template_ad_slots_script(mode, false, &settings, &slots, "/"); + + assert_eq!( + ran, did_not_run, + "{mode:?}: the template must be byte-identical whether or not the ad \ + stack ran; a cached object cannot carry one request's consent, bot, \ + prefetch or kill-switch decision" + ); + assert_eq!( + ran, None, + "{mode:?}: adSlots belongs in the per-request fragment, not the template" + ); + } + } + + #[test] + fn inline_mode_keeps_its_request_dependent_behaviour() { + // Inline responses are per-navigation and never shared, so gating is + // correct there. This guards against "fixing" the shared-mode bug by + // breaking the shipped path. + let settings = settings_with_slots(); + let slots = [slot()]; + + assert!( + template_ad_slots_script(AssemblyMode::Inline, true, &settings, &slots, "/") + .is_some(), + "inline should emit adSlots when the ad stack runs" + ); + assert_eq!( + template_ad_slots_script(AssemblyMode::Inline, false, &settings, &slots, "/"), + None, + "inline should emit nothing when the ad stack does not run" + ); + } + + #[test] + fn shared_modes_are_neutral_across_differing_slot_matches() { + // Slot matching folds in the request path. Under a shared mode even + // that must not reach the template. + let settings = settings_with_slots(); + + let matched = template_ad_slots_script( + AssemblyMode::Esi, + true, + &settings, + &[slot()], + "/news/article", + ); + let unmatched = + template_ad_slots_script(AssemblyMode::Esi, true, &settings, &[], "/other"); + + assert_eq!( + matched, unmatched, + "the template must not vary with slot matching under a shared mode" + ); + } + } + mod ssat_cache_policy_tests { use super::*; use crate::auction::provider::AuctionProvider; @@ -8127,6 +8286,7 @@ mod tests { auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, section_root: None, + assembly_mode: None, section_segment: None, slot: Vec::new(), } From 606543f51694bb081a6f53203377fdba25ca8411 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 20:07:28 +0530 Subject: [PATCH 180/395] Add the C2 shared-template cache gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 3 Step 3 of the #1009 ESI spike. No behaviour change: under the default Inline mode the gate reports InlineMode and does nothing. cache::core is not an HTTP cache. It stores whatever bytes it is handed and rejects nothing, so every safety condition belongs to the caller. c2_bypass_reason enumerates them rather than leaving them implicit: an authorized request, an origin Set-Cookie, a non-shareable Cache-Control, a non-200 status, and a non-HTML content type. Leak vectors are checked before mere ineligibility so an operator reading the log sees the security reason and not a content-type quibble. A DataDome block needs no separate detection — it replaces the document with a 403 and the status check covers it. There is a test saying so, because the next person will otherwise go looking for a marker that does not exist. Extracted is_uncacheable_by_cache_control into response_privacy rather than writing a third copy of the private/no-store predicate. It was already duplicated verbatim in both arms of the cookie-privacy net; this replaces both. The helper deliberately does not treat no-cache as disqualifying, because no-cache means revalidate before reuse rather than do not store, and the cookie-privacy net's reading is the correct one for HTTP. The C2 gate checks no-cache separately, as the stricter reading is right for a spike-owned cache we control. The gate has a real call site that logs its decision rather than an allow(dead_code). Clippy pushed back on the annotation and was right to: an #[expect] could not be satisfied in both the lib and test targets, and the honest answer was to wire it. Logging makes the decision observable during the spike instead of only once it starts mutating requests, and Authorization is captured before the origin send consumes the request. Verified: fmt, all six clippy targets, all four adapter suites, 1846 core tests. --- crates/trusted-server-core/src/publisher.rs | 267 ++++++++++++++++++ .../src/response_privacy.rs | 37 ++- 2 files changed, 289 insertions(+), 15 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 09d033d76..b708fdf05 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -2830,6 +2830,11 @@ pub async fn handle_publisher_request( } ); + // Recorded before the request is consumed by the origin send: the C2 gate + // below needs it, and an authorized response must never become a shared + // template. + let request_had_authorization = req.headers().contains_key(header::AUTHORIZATION); + if should_run_ad_stack { req.headers_mut().remove(header::IF_NONE_MATCH); req.headers_mut().remove(header::IF_MODIFIED_SINCE); @@ -2975,6 +2980,25 @@ pub async fn handle_publisher_request( .to_string(); let status = response.status(); + + // Evaluate the shared-template cache gate and log it. No behaviour change yet: + // the C2 read/write lands in Task 3 Step 4, and under the default `Inline` + // mode this reports `InlineMode` and logs nothing. Wiring it now gives the + // gate a real call site and makes the decision observable during the spike + // rather than only at the point it starts mutating requests. + if !matches!(assembly_mode, AssemblyMode::Inline) { + match c2_bypass_reason( + assembly_mode, + request_had_authorization, + status, + &content_type, + response.headers(), + ) { + Some(reason) => log::debug!("c2_template_cache bypass: {reason}"), + None => log::debug!("c2_template_cache eligible"), + } + } + let content_encoding = response .headers() .get(header::CONTENT_ENCODING) @@ -3560,6 +3584,88 @@ fn match_renderable_slots( /// /// Property names match what the client-side TSJS bundle expects: /// `gam_unit_path`, `div_id`, `formats`, and `targeting`. +/// Why a response must not enter the shared transformed-template cache (C2). +/// +/// `cache::core` is not an HTTP cache: it stores whatever bytes it is handed and +/// rejects nothing on its own. Every safety condition is the caller's to enforce, +/// so they are enumerated here rather than left implicit. +/// +/// Spike-only, for the #1009 ESI validation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub(crate) enum C2BypassReason { + /// Not a shared-template mode; there is no C2 object to write. + #[display("assembly mode is inline")] + InlineMode, + /// The origin set a cookie. Caching this would replay one visitor's cookie + /// to the next — and the cookie-privacy net downgrades *our* response, which + /// happens after the cache has already stored the origin's. + #[display("origin response carries Set-Cookie")] + OriginSetCookie, + /// The origin declared the response non-shareable. + #[display("origin marked the response private, no-store or no-cache")] + OriginNotShareable, + /// The request was authenticated. #1009 describes a Basic-Auth-gated + /// deployment, so an authorized response entering a shared cache is a live + /// concern rather than a hypothetical one. + #[display("request carried Authorization")] + AuthorizedRequest, + /// Not a 200. This is also what covers a `DataDome` block, which replaces the + /// document with a `403` (`integrations/datadome/protection.rs:778`). + #[display("status was not 200 OK")] + NonOkStatus, + /// Not HTML, so there is no template to transform. + #[display("content type is not text/html")] + NotHtml, +} + +/// Whether a response may be written to the shared transformed-template cache. +/// +/// Returns [`None`] when it is safe to cache, or the first disqualifying reason. +/// Leak vectors are checked before mere ineligibility so the reported reason is +/// the most serious one that applies. +/// +/// See `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +/// §6.6 for why C1, C2 and a final assembled-response cache are distinct, and why +/// the third must never exist. +pub(crate) fn c2_bypass_reason( + mode: AssemblyMode, + request_had_authorization: bool, + status: StatusCode, + content_type: &str, + response_headers: &edgezero_core::http::HeaderMap, +) -> Option { + if matches!(mode, AssemblyMode::Inline) { + return Some(C2BypassReason::InlineMode); + } + if request_had_authorization { + return Some(C2BypassReason::AuthorizedRequest); + } + if response_headers.contains_key(header::SET_COOKIE) { + return Some(C2BypassReason::OriginSetCookie); + } + // Reuse the cookie-privacy net's predicate rather than a third copy of it. + // That covers `private` and `no-store`; `no-cache` needs its own check + // because it means "revalidate before reuse", not "do not store" — a + // distinction that is correct for HTTP caches but too permissive for a + // spike-owned template cache, so treat it as disqualifying here. + let cache_control = response_headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .map(str::to_ascii_lowercase); + if crate::response_privacy::is_uncacheable_by_cache_control(response_headers) + || cache_control.is_some_and(|value| value.contains("no-cache")) + { + return Some(C2BypassReason::OriginNotShareable); + } + if status != StatusCode::OK { + return Some(C2BypassReason::NonOkStatus); + } + if !is_html_content_type(content_type) { + return Some(C2BypassReason::NotHtml); + } + None +} + /// What the `` seam injects, given the assembly mode. /// /// Under [`AssemblyMode::Inline`] the response is per-navigation and not shared, @@ -4582,6 +4688,167 @@ mod tests { .expect("should proxy publisher request") } + mod c2_gate_tests { + //! `cache::core` stores whatever it is handed and rejects nothing, so every + //! one of these conditions is the caller's to enforce. Each is a leak vector + //! or an eligibility rule, not a preference. + + use super::*; + use crate::creative_opportunities::AssemblyMode; + use edgezero_core::http::HeaderName; + + fn headers(pairs: &[(HeaderName, &str)]) -> edgezero_core::http::HeaderMap { + let mut map = edgezero_core::http::HeaderMap::new(); + for (name, value) in pairs { + map.insert( + name.clone(), + HeaderValue::from_str(value).expect("should build header value"), + ); + } + map + } + + fn shareable() -> edgezero_core::http::HeaderMap { + headers(&[(header::CACHE_CONTROL, "max-age=60")]) + } + + #[test] + fn a_plain_shareable_html_200_is_cacheable() { + for mode in [AssemblyMode::ClientFill, AssemblyMode::Esi] { + assert_eq!( + c2_bypass_reason(mode, false, StatusCode::OK, "text/html", &shareable()), + None, + "{mode:?}: a shareable HTML 200 should be eligible" + ); + } + } + + #[test] + fn inline_mode_never_writes_a_template() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Inline, + false, + StatusCode::OK, + "text/html", + &shareable() + ), + Some(C2BypassReason::InlineMode), + "inline has no shared template to write" + ); + } + + #[test] + fn an_authorized_request_is_never_cached() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + true, + StatusCode::OK, + "text/html", + &shareable() + ), + Some(C2BypassReason::AuthorizedRequest), + "an authenticated response must not enter a shared cache" + ); + } + + #[test] + fn an_origin_set_cookie_is_never_cached() { + let with_cookie = headers(&[ + (header::CACHE_CONTROL, "max-age=60"), + (header::SET_COOKIE, "sid=abc; Path=/"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + StatusCode::OK, + "text/html", + &with_cookie + ), + Some(C2BypassReason::OriginSetCookie), + "caching this would replay one visitor's cookie to the next" + ); + } + + #[test] + fn non_shareable_cache_control_is_refused_case_insensitively() { + for directive in [ + "private", + "no-store", + "no-cache", + "Private, max-age=60", + "NO-STORE", + "public, No-Cache", + ] { + let map = headers(&[(header::CACHE_CONTROL, directive)]); + assert_eq!( + c2_bypass_reason(AssemblyMode::Esi, false, StatusCode::OK, "text/html", &map), + Some(C2BypassReason::OriginNotShareable), + "`{directive}` should disqualify the response" + ); + } + } + + #[test] + fn a_datadome_block_is_refused_by_the_status_check() { + // DataDome replaces the document with a 403 + // (`integrations/datadome/protection.rs:778`). There is no separate + // marker to detect, and none is needed. + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + StatusCode::FORBIDDEN, + "text/html", + &shareable() + ), + Some(C2BypassReason::NonOkStatus), + "a blocked document must not become the shared template" + ); + } + + #[test] + fn non_html_is_refused() { + for content_type in ["text/x-component", "application/json", ""] { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + StatusCode::OK, + content_type, + &shareable() + ), + Some(C2BypassReason::NotHtml), + "`{content_type}` has no HTML template to transform" + ); + } + } + + #[test] + fn leak_vectors_are_reported_before_mere_ineligibility() { + // A response that fails several conditions should name the most serious + // one, so an operator reading the log sees the security reason rather + // than a content-type quibble. + let map = headers(&[ + (header::CACHE_CONTROL, "private"), + (header::SET_COOKIE, "sid=abc"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + true, + StatusCode::FORBIDDEN, + "application/json", + &map + ), + Some(C2BypassReason::AuthorizedRequest), + "authorization is the most serious disqualifier and should win" + ); + } + } + mod template_neutrality_tests { //! The gate for #1009's shared-template design. //! diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index e23348211..2d5fb31b4 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -9,7 +9,7 @@ //! cache such as Cloudflare would otherwise serve an operator/origin //! `Cache-Control: public` on a cookie-bearing response as-is. -use edgezero_core::http::{HeaderName, HeaderValue, Response, header}; +use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, Response, header}; use crate::settings::Settings; @@ -24,6 +24,25 @@ pub const CDN_CACHE_HEADERS: &[&str] = &[ "cloudflare-cdn-cache-control", ]; +/// Whether `Cache-Control` already forbids shared caching. +/// +/// Extracted because this predicate is needed in three places now: both arms of +/// the cookie-privacy net below, and the shared-template cache gate in +/// `publisher::c2_bypass_reason`. +/// +/// Directives are case-insensitive (RFC 9111 §5.2), so `No-Store` and `Private` +/// count. `no-cache` deliberately does **not**: it requires revalidation before +/// reuse, not a refusal to store, so a `no-cache` response is still shareable. +/// Callers needing the stricter reading must check it themselves. +#[must_use] +pub(crate) fn is_uncacheable_by_cache_control(headers: &HeaderMap) -> bool { + headers + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")) +} + /// Forces cookie-bearing responses to stay private to shared caches. /// /// Any response that sets a per-user cookie (notably the EC identity cookie) @@ -44,14 +63,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { for name in CDN_CACHE_HEADERS { response.headers_mut().remove(*name); } - // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match - // against a lowercased copy — `No-Store` / `Private` must count. - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let already_uncacheable = is_uncacheable_by_cache_control(response.headers()); if !already_uncacheable { response.headers_mut().insert( header::CACHE_CONTROL, @@ -77,12 +89,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: &mut Response) { enforce_set_cookie_cache_privacy(response); - let response_is_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let response_is_uncacheable = is_uncacheable_by_cache_control(response.headers()); for (key, value) in &settings.response_headers { if response_is_uncacheable From d9e059735f4ba7dc058b0496317959d8504f576a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 20:23:11 +0530 Subject: [PATCH 181/395] Decouple the body-close decision from the head script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes a defect the previous commit introduced. Gating the head seam on template neutrality made ad_slots_script None under the shared modes — and the body-close element handler read exactly that value to decide whether to inject at all. So shared modes silently stopped injecting anything at as a side effect of a change to . Safe, since emitting nothing cannot leak, but wrong for the reason the spec warns about: the gate has to be "did this response carry bids", not "does this page have slots". BodyCloseInjection replaces the inference with a named decision — None, InlineBids, or Marker — chosen by body_close_injection() at a site that knows the assembly mode. No new struct field was needed: settings is already threaded to all three processor-construction sites, so the mode is derivable there. Behaviour is unchanged. Inline still injects when slots matched and stays quiet when they did not. Esi deliberately returns None rather than a placeholder marker. The marker has to point at a fragment endpoint returning an executable script; /_ts/page-bids returns JSON and ESI splices fragment bytes verbatim, so aiming at it would put raw JSON where a script belongs. That endpoint does not exist yet, and a marker with nothing behind it is worse than no marker. A test pins the current answer so it changes deliberately rather than silently. The most useful test asserts body-close is identical whether or not the head script is present, under both shared modes. A decision that read the head script would be accidentally correct there today — because the head script is always absent under those modes — and wrong the moment that changes. Seven config literals in tests plus one in a benchmark now state their intent explicitly instead of relying on the old inference, which is the improvement rather than a cost. clippy --all-targets caught the benchmark; test runs alone did not. Verified: fmt, all six clippy targets, all four adapter suites, 1850 core tests. --- .../benches/html_processor_bench.rs | 8 +- .../trusted-server-core/src/html_processor.rs | 74 +++++++++-- crates/trusted-server-core/src/publisher.rs | 115 +++++++++++++++++- 3 files changed, 186 insertions(+), 11 deletions(-) diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 96eec2f1f..e13b9fdb3 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -1,5 +1,7 @@ use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; -use trusted_server_core::html_processor::{HtmlProcessorConfig, create_html_processor}; +use trusted_server_core::html_processor::{ + BodyCloseInjection, HtmlProcessorConfig, create_html_processor, +}; use trusted_server_core::integrations::IntegrationRegistry; use trusted_server_core::streaming_processor::StreamProcessor as _; @@ -13,6 +15,10 @@ fn make_config() -> HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + // The benchmark measures URL rewriting, not ad injection, and + // `ad_slots_script` is `None` here — matching the previous behaviour, + // which inferred no body-close work from that. + body_close: BodyCloseInjection::None, } } diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 889234b56..f0d21a7ad 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -155,6 +155,30 @@ impl StreamProcessor for HtmlWithPostProcessing { fn reset(&mut self) {} } +/// What the `` seam injects. +/// +/// This is a decision, not a side effect of whether the `` script exists. +/// An earlier shape gated body-close injection on `ad_slots_script.is_some()`, +/// which coupled two independent choices: once a shared-template mode stopped +/// emitting the head script, body-close injection silently stopped too. +/// +/// See `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +/// §6.7. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum BodyCloseInjection { + /// Emit nothing. Either no slots matched under the inline path, or a + /// client-fill mode where the browser fetches the fragment unprompted. + #[default] + None, + /// Read the auction result from `ad_bids_state` and inject it, falling back to + /// an empty payload. Today's shipped behaviour. + InlineBids, + /// Emit this markup verbatim — an `` for the edge to assemble. + /// Must be identical for every request that reaches the transform, or the + /// cached template is not shared-safe. + Marker(String), +} + /// Configuration for HTML processing #[derive(Clone)] pub struct HtmlProcessorConfig { @@ -175,6 +199,9 @@ pub struct HtmlProcessorConfig { pub max_buffered_body_bytes: usize, /// Request-scoped conditional diagnostics delivery decision. pub gpt_diagnostics: Option, + /// What the `` seam injects. Decided by the caller rather than inferred + /// from [`Self::ad_slots_script`]. + pub body_close: BodyCloseInjection, } impl HtmlProcessorConfig { @@ -196,6 +223,7 @@ impl HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: settings.publisher.max_buffered_body_bytes, gpt_diagnostics: None, + body_close: BodyCloseInjection::None, } } @@ -217,6 +245,17 @@ impl HtmlProcessorConfig { self } + /// Set what the `` seam injects. + /// + /// Separate from [`with_ad_state`](Self::with_ad_state) because the two are + /// independent decisions: a shared-template mode emits no head script and + /// still needs a body-close marker. + #[must_use] + pub fn with_body_close(mut self, body_close: BodyCloseInjection) -> Self { + self.body_close = body_close; + self + } + /// Attach the request-scoped conditional diagnostics decision. #[must_use] pub fn with_gpt_diagnostics(mut self, decision: Option) -> Self { @@ -304,6 +343,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let integration_registry = config.integrations.clone(); let script_rewriters = integration_registry.script_rewriters(); let ad_slots_script = config.ad_slots_script.clone(); + let body_close = config.body_close.clone(); let ad_bids_state = config.ad_bids_state.clone(); let gpt_diagnostics = config.gpt_diagnostics.clone(); @@ -371,25 +411,38 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso element!("body", { let state = ad_bids_state.clone(); let injected_bids = injected_bids.clone(); - let has_slots = ad_slots_script.is_some(); + let body_close = body_close.clone(); move |el| { - if !has_slots { + if matches!(body_close, BodyCloseInjection::None) { return Ok(()); } let state = state.clone(); let injected_bids = injected_bids.clone(); + let body_close = body_close.clone(); if let Some(handlers) = el.end_tag_handlers() { let handler: EndTagHandler<'static> = Box::new(move |end_tag: &mut EndTag<'_>| { if injected_bids.swap(true, Ordering::SeqCst) { return Ok(()); } - let script_guard = state.lock().expect("should lock bid state"); - let bids_script = match &*script_guard { - Some(s) => s.clone(), - None => build_empty_bids_script(), + let markup = match &body_close { + // Verbatim, and identical on every request that + // reaches the transform — that is what makes the + // cached template shared-safe. + BodyCloseInjection::Marker(marker) => marker.clone(), + BodyCloseInjection::InlineBids => { + let script_guard = state.lock().expect("should lock bid state"); + match &*script_guard { + Some(s) => s.clone(), + None => build_empty_bids_script(), + } + } + // Unreachable: the element handler returned early + // above. Kept exhaustive rather than using `_` so a + // new variant is a compile error here. + BodyCloseInjection::None => return Ok(()), }; - end_tag.before(&bids_script, ContentType::Html); + end_tag.before(&markup, ContentType::Html); Ok(()) }); handlers.push(handler); @@ -684,6 +737,7 @@ mod tests { fn create_test_config() -> HtmlProcessorConfig { HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_owned(), request_host: "test.example.com".to_owned(), request_scheme: "https".to_owned(), @@ -1528,6 +1582,7 @@ mod tests { #[test] fn injects_ad_slots_at_head_open() { let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1603,6 +1658,7 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1639,6 +1695,7 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1676,6 +1733,7 @@ mod tests { let request_host = "proxy.test-publisher.example.com"; let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.test-publisher.example.com".to_string(), request_host: request_host.to_string(), request_scheme: "https".to_string(), @@ -1727,6 +1785,7 @@ mod tests { // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1756,6 +1815,7 @@ mod tests { // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index b708fdf05..e789f5dce 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -55,6 +55,7 @@ use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; +use crate::html_processor::BodyCloseInjection; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; use crate::integrations::IntegrationRegistry; use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; @@ -948,6 +949,39 @@ struct HtmlStreamProcessorParams<'a> { gpt_diagnostics: Option, } +/// What the `` seam should inject, given the assembly mode. +/// +/// Explicit rather than inferred. The previous shape read +/// `ad_slots_script.is_some()` inside the element handler, which silently coupled +/// two independent decisions: once [`template_ad_slots_script`] stopped emitting a +/// head script under a shared mode, body-close injection stopped with it. +/// +/// `Esi` returns [`BodyCloseInjection::None`] for now rather than a placeholder +/// marker. The marker must point at a dedicated fragment endpoint returning an +/// executable script — `/_ts/page-bids` returns JSON, and ESI splices fragment +/// bytes verbatim, so aiming at it would put raw JSON where a script belongs. +/// That endpoint does not exist yet, and emitting a marker with nothing behind it +/// would be worse than emitting nothing. +pub(crate) fn body_close_injection( + mode: AssemblyMode, + head_script_present: bool, +) -> BodyCloseInjection { + match mode { + // Per-navigation and never shared, so gating on slot presence is correct. + AssemblyMode::Inline => { + if head_script_present { + BodyCloseInjection::InlineBids + } else { + BodyCloseInjection::None + } + } + // The browser fetches the fragment unprompted; nothing to emit. + AssemblyMode::ClientFill => BodyCloseInjection::None, + // Pending the fragment endpoint. See the note above. + AssemblyMode::Esi => BodyCloseInjection::None, + } +} + fn create_html_stream_processor( params: HtmlStreamProcessorParams<'_>, ) -> Result, Report> { @@ -959,9 +993,20 @@ fn create_html_stream_processor( params.origin_host, params.request_host, params.request_scheme, - ) - .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(params.gpt_diagnostics); + ); + + let assembly_mode = params + .settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::assembly_mode) + .unwrap_or_default(); + let body_close = body_close_injection(assembly_mode, params.ad_slots_script.is_some()); + + let config = config + .with_ad_state(params.ad_slots_script, params.ad_bids_state) + .with_gpt_diagnostics(params.gpt_diagnostics) + .with_body_close(body_close); Ok(create_html_processor(config)) } @@ -4688,6 +4733,70 @@ mod tests { .expect("should proxy publisher request") } + mod body_close_decision_tests { + //! The `` decision must not be inferred from the `` script. + //! + //! Coupling them is a live defect, not a hypothetical: gating the head seam + //! on template neutrality made `ad_slots_script` `None` under shared modes, + //! which silently disabled body-close injection too. These tests pin the two + //! decisions apart. + + use super::*; + use crate::creative_opportunities::AssemblyMode; + + #[test] + fn inline_injects_bids_only_when_the_head_script_is_present() { + assert_eq!( + body_close_injection(AssemblyMode::Inline, true), + BodyCloseInjection::InlineBids, + "inline with matched slots should inject the auction result" + ); + assert_eq!( + body_close_injection(AssemblyMode::Inline, false), + BodyCloseInjection::None, + "inline without matched slots should leave the publisher's flow alone" + ); + } + + #[test] + fn shared_modes_do_not_depend_on_the_head_script() { + // The decision must be the same either way. Under a shared mode the head + // script is always absent, so a decision that read it would be + // accidentally correct here and wrong the moment that changes. + for mode in [AssemblyMode::ClientFill, AssemblyMode::Esi] { + assert_eq!( + body_close_injection(mode, true), + body_close_injection(mode, false), + "{mode:?}: body-close must not vary with head-script presence" + ); + } + } + + #[test] + fn client_fill_emits_nothing_because_the_browser_fetches_unprompted() { + assert_eq!( + body_close_injection(AssemblyMode::ClientFill, false), + BodyCloseInjection::None + ); + } + + #[test] + fn esi_emits_nothing_until_the_fragment_endpoint_exists() { + // Deliberately not a placeholder marker. `/_ts/page-bids` returns JSON + // and ESI splices fragment bytes verbatim, so pointing at it would put + // raw JSON where an executable script belongs. Emitting a marker with + // nothing behind it is worse than emitting nothing. + // + // This test is expected to change when that endpoint lands — it exists + // to make that a deliberate edit rather than a silent one. + assert_eq!( + body_close_injection(AssemblyMode::Esi, false), + BodyCloseInjection::None, + "Esi should emit nothing until a script-returning fragment endpoint exists" + ); + } + } + mod c2_gate_tests { //! `cache::core` stores whatever it is handed and rejects nothing, so every //! one of these conditions is the caller's to enforce. Each is a leak vector From f0ab7ac7648c7787d0b9348ad09bc59e2efe2751 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 20:51:27 +0530 Subject: [PATCH 182/395] Record Task 3 implementation progress and its limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steps 1, 2, 2b and 3 are done and behaviour-neutral under the default Inline mode. Step 2c (emit the Esi marker) and Step 4 (the cache read/write) are not, and the record says why rather than leaving them looking merely unstarted: the marker needs a fragment endpoint returning an executable script, and Step 4 is blocked on a design choice the plan deliberately defers. Records the defect this work introduced and then caught. Gating the head seam on neutrality made ad_slots_script None under shared modes, and the body-close handler read that value to decide whether to inject at all — so shared modes silently stopped injecting at as a side effect of a change. Found by reading the handler while starting the next step, not by a failing test. It is the same shape as the bug the whole task exists to prevent: something that looks correct and quietly does nothing. Also records what the coverage does not cover. Fourteen tests prove tsjs.adSlots is request-neutral. They say nothing about the other things injected at the same seam — integration head_inserts, the gpt-diagnostics bootstrap, the RSC placeholder rewriter — which the spec flags for audit and which is still outstanding. Request-neutrality is asserted for one element, not established for the template, and reading the test names would suggest otherwise. And a gate note: clippy --all-targets caught a benchmark construction site that all four test suites missed. --- .../2026-08-08-1009-measurement-findings.md | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md index 9e68acb27..b12d115a6 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -217,6 +217,66 @@ Compiling and a cache round-trip are not an implementation. Nothing yet exercise `lol_html` transform into C2, ESI assembly, or any runtime behaviour on the publisher path, and the `esi` dependency is added but unused. +## ESI spike Task 3 — implementation progress + +**Date:** 2026-08-10. All of it behaviour-neutral under the default +`AssemblyMode::Inline`; nothing here changes a shipped code path. + +| Step | State | +| -------------------------------- | ------------------------------------------------------------------- | +| 1 — `AssemblyMode` setting | **Done.** `Option` on `CreativeOpportunitiesConfig`. | +| 2 — head-seam neutrality gate | **Done.** `template_ad_slots_script`, three byte-identity tests. | +| 2b — body-close decoupling | **Done.** `BodyCloseInjection`, `body_close_injection`. | +| 2c — emit the marker under `Esi` | **Not done.** Blocked on the fragment endpoint; see below. | +| 3 — C2 eligibility gate | **Done.** `c2_bypass_reason`, eight tests. Logs only, no cache I/O. | +| 4 — C2 cache read/write | **Not started.** Design choice open; see below. | + +### What is deliberately absent + +**No marker is emitted under `Esi`.** The marker must point at a fragment endpoint +returning an **executable script**. `/_ts/page-bids` returns JSON +(`publisher.rs`, `handle_page_bids`) and ESI splices fragment bytes verbatim, so aiming +at it would put raw JSON where a script belongs. That endpoint does not exist, and a +marker with nothing behind it is worse than no marker. A test pins the current answer so +it changes deliberately. + +**No cache read or write.** `c2_bypass_reason` has a real call site that logs its verdict, +which makes the decision observable during the spike without mutating anything. Task 3 +Step 4 is blocked on choosing between read-through-with-body-transform and explicit +`cache::core` — the plan names that as a decision to make before writing code, and it is +under investigation rather than assumed. + +### A defect this work introduced and then caught + +Gating the head seam on neutrality made `ad_slots_script` `None` under the shared modes. +The body-close element handler read exactly that value to decide whether to inject at all, +so shared modes silently stopped injecting anything at `` — a side effect of a +`` change. Safe, since emitting nothing cannot leak, but wrong in the way the spec +warns about: the gate has to be "did this response carry bids", not "does this page have +slots". + +Found by reading the handler while starting the next step, not by a failing test. Fixed by +replacing the inference with a named decision. The test that now guards it asserts +body-close is identical whether or not the head script is present — a decision that read +the head script would be _accidentally_ correct today, because that script is always +absent under shared modes, and wrong the moment that changes. + +Worth recording because it is the same shape as the bug the whole task exists to prevent: +something that looks correct and quietly does nothing. + +### Coverage and its limits + +Fourteen new tests. `fmt`, all six clippy targets, and all four adapter suites pass, with +1850 core tests under Viceroy. `clippy --all-targets` caught a benchmark construction site +that all four test suites missed — the suites are not the whole gate. + +**The neutrality guarantee is narrower than it looks.** The tests prove `tsjs.adSlots` is +neutral. They say nothing about the other things injected at the same seam — integration +`head_inserts`, the gpt-diagnostics bootstrap, the RSC placeholder rewriter — which the +spec flags as needing an audit and which that audit has not yet covered. Until it does, +treat request-neutrality as asserted for one element rather than established for the +template. + ## Step B — consumers of TS's own response headers Not yet run. From 672836c1a8a0498ea1d33ec7312a3d9e92ac2ce4 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 20:59:46 +0530 Subject: [PATCH 183/395] Decide the C2 cache design: cache::core, not read-through MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plan left this open between fastly::cache::core and read-through caching with after_send plus set_body_transform. Investigated and verified against the pinned SDK and Viceroy 0.17 source. Read-through is not viable here, on three hard blockers rather than on preference. Viceroy stubs the entire HTTP Cache ABI, and the SDK converts that into a send error rather than a fallback: is_request_cacheable returns NotAvailable, which makes must_use_host_caching true, which with a send hook set returns HttpCacheApiUnsupported. Setting after_send therefore makes every publisher origin fetch fail under fastly compute serve, cargo test-fastly, and the parity suite. The whole local loop dies. with_cache_bypass makes the hook silently dead anyway. get_caching_mode checks cache_override.is_pass() first and returns host caching, so after_send is never invoked and no error is raised — on exactly the requests in scope, quietly. And the closure bounds are incompatible with this codebase. with_after_send requires Fn + Send + Sync + 'static, while everything the rewriter needs is !Send by construction, which is why the platform layer is async_trait(?Send) throughout. set_body_transform is also synchronous and so could never await the auction collect. Recorded rather than merely chosen, because read-through's appeal is real — CandidateResponse::apply_and_stream_back is execute_and_stream_back with HTTP semantics attached — and someone will otherwise propose it again. Also settled: core cannot reach it at all, since PlatformHttpRequest has no callback slot and adding one would name Fastly types in portable core. Adds the exact insertion point, the one required hoist, and four risks the investigation surfaced that are specific to this codebase: Vary is in the key list but c2_bypass_reason does not check it; store bytes plus a metadata envelope and rebuild every header on a hit rather than replaying origin headers into a path that strips them; Content-Encoding and host/scheme both belong in the key. Plus a follow-up to file rather than fix: the auction is dispatched before the lookup, so under the shared modes it is already pure waste. Tee-ing turns out to be unnecessary. With any post-processor registered — and Next.js always registers one — the transformed document arrives as one contiguous buffer, so it is two write_all calls on the same slice. Keep execute_and_stream_back for transaction correctness and request collapsing, not for memory. Corrects the findings document: Viceroy implements purge_surrogate_key against the same in-process cache, so C2's purge-based rollback is locally testable. C1's, which is what Stage 0 exposes, still is not. --- .../2026-08-08-1009-measurement-findings.md | 6 + .../2026-08-10-1009-esi-validation-spike.md | 118 ++++++++++++++++-- 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md index b12d115a6..0b6a70f67 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -147,6 +147,12 @@ surrogate keys the _origin_ supplies on its responses, or the HTTP cache's own request/candidate surrogate-key surface. Confirm which is available before relying on it — an earlier revision of this document conflated the two. +**C2's purge is locally testable; C1's is not.** Verified 2026-08-10: Viceroy 0.17 +implements `purge_surrogate_key` against the same in-process cache it serves reads from +(`viceroy-lib-0.17.0/src/wiggle_abi/fastly_purge_impl.rs:10-32`), soft purge included. So +the purge-based rollback for the C2 template cache the spike builds can be exercised end +to end without a Fastly service. That does nothing for Stage 0, whose exposure is C1. + Rollback is therefore: flip the flag, **then** purge C1 by whichever mechanism is actually available (or roll a versioned key namespace), **then** observe past the origin TTL before declaring the incident closed. With no C1 purge path wired, the tail is the TTL itself — diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index f1f3776f5..1b19bc132 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -468,16 +468,114 @@ built from — and decide explicitly whether the stored template is compressed. Per-user signals must never appear in the key. If a signal cannot be excluded from the template, it does not belong in C2 at all. -**Design choice to make explicitly before writing code.** Two viable shapes: - -1. **Read-through with `after_send` + `set_body_transform`** — keeps HTTP semantics, - revalidation, and stale handling for free; less control over the key. -2. **`cache::core` as above** — full control; you own metadata, revalidation, and the - stale state machine. - -This plan assumes (2). If (1) is chosen, Step 4 is rewritten and the metadata envelope -disappears. Either way, the platform boundary must sit **before** the origin request, or -a C2 HIT cannot actually skip the fetch — which is the entire point. +### Design decided 2026-08-10: `cache::core`. Do not revisit read-through. + +An earlier revision left this open between `cache::core` and read-through caching with +`after_send` + `set_body_transform`. Investigated and verified against the pinned SDK and +Viceroy 0.17 source. **Read-through is not viable here** — not on preference, on three +hard blockers: + +1. **Viceroy stubs the entire HTTP Cache ABI**, and the SDK converts that into a _send + error_ rather than a fallback. `is_request_cacheable` returns + `Err(NotAvailable("HTTP Cache API primitives"))` + (`viceroy-lib-0.17.0/src/wiggle_abi/http_cache.rs:108-114`; 26 such stubs in that + file), which makes `must_use_host_caching()` true, which with a send hook set returns + `Err(SendErrorCause::HttpCacheApiUnsupported)` + (`fastly-0.12.1/src/http/request.rs:626-632`). **Setting `after_send` makes every + publisher origin fetch fail** under `fastly compute serve`, `cargo test-fastly`, and + the parity suite. The whole local loop dies. +2. **`with_cache_bypass` makes the hook silently dead.** `get_caching_mode` checks + `cache_override.is_pass()` **first** (`request.rs:612-615`) and returns host caching, so + `after_send` is never invoked and no error is raised. On exactly the requests in scope, + today, the hook would do nothing quietly. +3. **The closure bounds are incompatible with this codebase.** `with_after_send` requires + `Fn + Send + Sync + 'static` (`request.rs:545-550`). Everything the rewriter needs is + `!Send` by construction — `edgezero_core::body::Body` wraps a `LocalBoxStream` + deliberately, which is why the platform layer is `#[async_trait(?Send)]` throughout. + And `set_body_transform` is synchronous, so it could never await the auction collect. + +Read-through's appeal was real — `CandidateResponse::apply_and_stream_back` is +`execute_and_stream_back` with HTTP semantics attached, and TTL/SWR/vary/surrogate keys +derived from origin headers for free. It is simply unreachable from here. + +**Also settled: core cannot reach it at all.** `PlatformHttpRequest` +(`platform/http.rs:16-37`) is a plain data struct with no callback slot, and carrying one +would name `fastly::http::CandidateResponse` in portable core, breaking the other three +adapters. + +### Follow the existing null-object pattern + +`cache::core` fits the shape the repo already uses four times for a Fastly-only capability +behind a portable trait: `UnavailableHttpClient` (`platform/http.rs:216-243`), +`UnavailableKvStore` (`platform/kv.rs:14-17`), and the `RuntimeServices.kv_store` +field/accessor/builder (`platform/types.rs:170,222,269,330`). Add +`PlatformTemplateCache` the same way, and follow +`crates/trusted-server-adapter-fastly/src/ec_kv.rs` — 140 lines, the repo's only real +edge-storage read/write — rather than inventing a shape. + +**Return `EdgeBody`, not `Vec`.** `EdgeBody::Stream` exists, +`fastly_body_to_edge_stream` (`adapter-fastly/src/platform.rs:503`) already converts, and +`PublisherResponse::Buffered` tolerates a live stream (`publisher.rs:1019-1022`). + +### Exact insertion point + +**Immediately before `let mut platform_request = PlatformHttpRequest::new(...)`** — the +last line before `req` is consumed, and a few lines before the origin send. Everything +needed is in scope there: `settings`, `services`, the final URI and Host, `backend_name`, +`request_path`, `matched_slots`, `should_run_ad_stack`, `request_had_authorization`, +`request_host`, `request_scheme`. + +**One required move:** `assembly_mode` is currently computed _after_ the send, for the +logging call site. It depends only on `settings`, so hoist it above the insertion point. + +**Tee-ing is not needed.** With any post-processor registered — and the Next.js +integration always registers one — `HtmlWithPostProcessing` emits nothing until the final +chunk and then returns the whole transformed document as one contiguous buffer +(`html_processor.rs:92-97,148`). Two `write_all` calls on the same slice; no tee +abstraction, no extra copy. Still use `execute_and_stream_back`, but for transaction +correctness and request collapsing rather than for memory. On a hit the processor is never +built at all. + +- [ ] **Step 4b: close the risks the design investigation surfaced** + +Four, all specific to this codebase rather than to `cache::core` in general. + +**`Vary` is in the key list but nothing consumes it.** `c2_bypass_reason` checks +`Set-Cookie`, `Cache-Control`, `Authorization`, status and content type — **not `Vary`**. +Viceroy supports `WriteOptions.vary_rule`, so the mechanism exists; the gate has to use +it. Until then the key is missing a signal the origin explicitly declares, and Step A's +verdict is a `PROVISIONAL PASS`, not a release gate. + +**Store bytes plus a metadata envelope; rebuild every header on a hit.** The publisher +path forces `private, no-store` and strips `ETag`/`Last-Modified`/CDN headers _after_ the +send. Replaying stored origin headers would fight that. Store only the transformed body +and a small `user_metadata` envelope — content encoding, content type, schema version, +tsjs hash — and construct every response header from scratch on a hit. Then no origin +header is ever replayed and the `Set-Cookie` privacy net is trivially safe. +`get_user_metadata` is implemented in Viceroy. + +**Content-Encoding belongs in the key.** The streaming pipeline pairs input encoding to +the same output encoding, so the transformed bytes inherit whatever the origin negotiated +from the client's `Accept-Encoding` — still gzip, deflate, br or identity after +`restrict_accept_encoding` narrows it. Either key on the negotiated encoding or normalize +to identity in the cache and re-encode on read. Getting this wrong serves brotli bytes to +a client that asked for gzip. + +**Host and scheme belong in the key.** The post-processed output is host-dependent by +construction: `request_host` and `request_scheme` reach `IntegrationHtmlContext`. + +- [ ] **Step 4c: file the wasted-dispatch follow-up** + +The auction is dispatched _before_ the insertion point. Under `Esi` and `ClientFill` the +root injects nothing, so that dispatch is already pure waste on this branch — and on a C2 +hit it is waste that must be cleaned up via `emit_abandoned_auction` or it leaks +telemetry. + +Keeping the lookup at the insertion point above is right for the spike: minimal diff, and +lookup latency overlaps the in-flight auction. Moving it earlier would eliminate the +wasted dispatch but serialize the lookup ahead of dispatch. **File it; do not fix it +here.** Suppressing root-level dispatch under the shared modes is Task 4's job, where it +also has to be reconciled with the exactly-one-auction gate. - [ ] **Step 5: Unit tests, then the target suite** From fee46220a0165d980d3583b6f2ca056402701d81 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 10 Aug 2026 21:14:46 +0530 Subject: [PATCH 184/395] Guard the silent-waste and cookie gaps a review found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from a code review of the four preceding commits. Two are fixed here; the third waits on an audit that is still running. The auction dispatch was never gated on AssemblyMode. assembly_mode was computed after the dispatch decision, so flipping to client_fill or esi today would still send real SSP bid requests, hold the response for the full auction budget, and then discard the result — because both injection seams now return None — with no error, no warning and no log. That is precisely the silent-waste signature §5 of the design doc is about, reached by an incomplete feature flag rather than by removing the hold. assembly_mode is hoisted above the dispatch, which the C2 design investigation wanted anyway, and root_auction_is_useful gates it. The interesting test there does not assert per-variant. It derives the invariant: a root auction is useful exactly when a seam will consume its result. A new mode cannot make the dispatch gate and the injection decisions disagree without failing it. c2_bypass_reason omitted the forwarded client Cookie, which the design doc's own §4 names as a leak vector and the plan's checklist also missed. TS forwards client cookies to origin unchanged with no strip on the publisher path, so a response can be cookie-personalized while carrying no Set-Cookie itself, having no Cache-Control at all, and being a 200 HTML — every other condition reports it cacheable. Now disqualifying until an origin Vary covering Cookie is verified. The test uses exactly that shape rather than a response that would fail some other condition anyway. Also folds the duplicated Cache-Control lookup into one pass. The previous version built a lowercased copy and then called is_uncacheable_by_cache_control, which re-fetched and re-lowercased the same header. Not fixed here: the head seam still injects integration head_inserts and the gpt-diagnostics bootstrap unconditionally, so request-neutrality is asserted for adSlots only. It happens not to leak today because gpt_diagnostics::finalize_response stamps private/no-store before the C2 gate reads headers — a load-bearing coincidence that is undocumented and untested. A neutrality audit covering that seam is still in flight; fixing it on partial information would mean doing it twice. Verified: fmt, all six clippy targets, all four adapter suites, 1853 core tests. --- crates/trusted-server-core/src/publisher.rs | 165 ++++++++++++++++++-- 1 file changed, 148 insertions(+), 17 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index e789f5dce..b8a035e2c 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -949,6 +949,24 @@ struct HtmlStreamProcessorParams<'a> { gpt_diagnostics: Option, } +/// Whether a root-level auction has any consumer under this assembly mode. +/// +/// Only [`AssemblyMode::Inline`] injects the auction result into the root document. +/// Under the shared-template modes both seams emit nothing, so a dispatched root +/// auction would bill the SSPs, hold the response for the full budget, and have its +/// result discarded with no error and no log. +/// +/// This is the guard for the failure mode described in §5 of +/// `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md`, +/// reached here by an incomplete feature flag rather than by removing the hold. +pub(crate) fn root_auction_is_useful(mode: AssemblyMode) -> bool { + match mode { + AssemblyMode::Inline => true, + // The fragment path runs its own auction; see the spike plan's Task 4. + AssemblyMode::ClientFill | AssemblyMode::Esi => false, + } +} + /// What the `` seam should inject, given the assembly mode. /// /// Explicit rather than inferred. The previous shape read @@ -2742,9 +2760,27 @@ pub async fn handle_publisher_request( // dispatch_auction returns — DispatchedAuction holds no lifetime — so req // can be mutated and sent to origin immediately after. let mut auction_observation: Option = None; + let assembly_mode = settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::assembly_mode) + .unwrap_or_default(); + let mut auction_request_for_telemetry: Option = None; let mut dispatched_auction = if matched_slots.is_empty() { None + } else if !root_auction_is_useful(assembly_mode) { + // Shared-template modes inject nothing at the root: `template_ad_slots_script` + // and `body_close_injection` both return `None`. Dispatching here would send + // real SSP requests, hold the response for the full auction budget, and then + // discard the result with no error and no log — the silent-waste signature + // §5 of the design doc is entirely about. The fragment path runs its own + // auction; this one has no consumer. + log::debug!( + "skipping root auction dispatch: assembly mode {assembly_mode:?} injects \ + nothing at the root" + ); + None } else { // Telemetry attribution must use the same publisher identity as the // outbound bid request. On the navigation path `request_host` is the @@ -2879,6 +2915,7 @@ pub async fn handle_publisher_request( // below needs it, and an authorized response must never become a shared // template. let request_had_authorization = req.headers().contains_key(header::AUTHORIZATION); + let request_had_cookie = req.headers().contains_key(header::COOKIE); if should_run_ad_stack { req.headers_mut().remove(header::IF_NONE_MATCH); @@ -2968,11 +3005,6 @@ pub async fn handle_publisher_request( crate::integrations::gpt_diagnostics::finalize_response(&gpt_diagnostics, &mut response); - let assembly_mode = settings - .creative_opportunities - .as_ref() - .map(CreativeOpportunitiesConfig::assembly_mode) - .unwrap_or_default(); let ad_slots_script = template_ad_slots_script( assembly_mode, should_run_ad_stack, @@ -3035,6 +3067,7 @@ pub async fn handle_publisher_request( match c2_bypass_reason( assembly_mode, request_had_authorization, + request_had_cookie, status, &content_type, response.headers(), @@ -3661,6 +3694,15 @@ pub(crate) enum C2BypassReason { /// Not HTML, so there is no template to transform. #[display("content type is not text/html")] NotHtml, + /// The request carried a `Cookie`, which TS forwards to origin unchanged — there + /// is no `Cookie` strip on the publisher path. Cookie-personalized HTML is + /// therefore cross-servable unless the origin declares `Vary: Cookie` or marks + /// those responses private, and a response can be personalized without carrying + /// `Set-Cookie` itself when the session was established earlier. Named in §4 of + /// the design doc; disqualifying until the origin's `Vary` is verified to cover + /// it. + #[display("request carried Cookie and the origin's Vary does not cover it")] + CookieForwarded, } /// Whether a response may be written to the shared transformed-template cache. @@ -3675,6 +3717,7 @@ pub(crate) enum C2BypassReason { pub(crate) fn c2_bypass_reason( mode: AssemblyMode, request_had_authorization: bool, + request_had_cookie: bool, status: StatusCode, content_type: &str, response_headers: &edgezero_core::http::HeaderMap, @@ -3685,21 +3728,24 @@ pub(crate) fn c2_bypass_reason( if request_had_authorization { return Some(C2BypassReason::AuthorizedRequest); } + if request_had_cookie { + return Some(C2BypassReason::CookieForwarded); + } if response_headers.contains_key(header::SET_COOKIE) { return Some(C2BypassReason::OriginSetCookie); } - // Reuse the cookie-privacy net's predicate rather than a third copy of it. - // That covers `private` and `no-store`; `no-cache` needs its own check - // because it means "revalidate before reuse", not "do not store" — a - // distinction that is correct for HTTP caches but too permissive for a - // spike-owned template cache, so treat it as disqualifying here. - let cache_control = response_headers + // One pass over the header. `private` and `no-store` match the cookie-privacy + // net's reading; `no-cache` is added because it means "revalidate before reuse" + // rather than "do not store" — correct for an HTTP cache, too permissive for a + // spike-owned one. + let non_shareable = response_headers .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()) - .map(str::to_ascii_lowercase); - if crate::response_privacy::is_uncacheable_by_cache_control(response_headers) - || cache_control.is_some_and(|value| value.contains("no-cache")) - { + .map(str::to_ascii_lowercase) + .is_some_and(|value| { + value.contains("private") || value.contains("no-store") || value.contains("no-cache") + }); + if non_shareable { return Some(C2BypassReason::OriginNotShareable); } if status != StatusCode::OK { @@ -4733,6 +4779,50 @@ mod tests { .expect("should proxy publisher request") } + mod root_auction_gate_tests { + //! Guards the silent-waste failure mode: dispatching an auction whose result + //! nothing will consume. Under the shared modes both injection seams emit + //! nothing, so a dispatched root auction bills the SSPs, holds the response + //! for the full budget, and discards the result with no error and no log. + + use super::*; + use crate::creative_opportunities::AssemblyMode; + + #[test] + fn only_inline_has_a_consumer_for_a_root_auction() { + assert!( + root_auction_is_useful(AssemblyMode::Inline), + "inline injects the auction result at ``" + ); + for mode in [AssemblyMode::ClientFill, AssemblyMode::Esi] { + assert!( + !root_auction_is_useful(mode), + "{mode:?}: neither seam injects, so a root auction has no consumer" + ); + } + } + + #[test] + fn the_gate_agrees_with_the_injection_decisions() { + // The real invariant: a root auction is useful exactly when something + // will read it. Deriving that from the two seam decisions rather than + // asserting it per-variant means a new mode cannot make these disagree. + for mode in [ + AssemblyMode::Inline, + AssemblyMode::ClientFill, + AssemblyMode::Esi, + ] { + let something_consumes_it = + body_close_injection(mode, true) != BodyCloseInjection::None; + assert_eq!( + root_auction_is_useful(mode), + something_consumes_it, + "{mode:?}: dispatch usefulness must track whether a seam consumes the result" + ); + } + } + } + mod body_close_decision_tests { //! The `` decision must not be inferred from the `` script. //! @@ -4825,7 +4915,14 @@ mod tests { fn a_plain_shareable_html_200_is_cacheable() { for mode in [AssemblyMode::ClientFill, AssemblyMode::Esi] { assert_eq!( - c2_bypass_reason(mode, false, StatusCode::OK, "text/html", &shareable()), + c2_bypass_reason( + mode, + false, + false, + StatusCode::OK, + "text/html", + &shareable() + ), None, "{mode:?}: a shareable HTML 200 should be eligible" ); @@ -4838,6 +4935,7 @@ mod tests { c2_bypass_reason( AssemblyMode::Inline, false, + false, StatusCode::OK, "text/html", &shareable() @@ -4853,6 +4951,7 @@ mod tests { c2_bypass_reason( AssemblyMode::Esi, true, + false, StatusCode::OK, "text/html", &shareable() @@ -4862,6 +4961,27 @@ mod tests { ); } + #[test] + fn a_forwarded_request_cookie_disqualifies_even_without_set_cookie() { + // The dangerous case: session established on an earlier request, so this + // response carries no Set-Cookie, has no Cache-Control at all, is a 200, + // and is HTML — yet is personalized because TS forwarded the Cookie to + // origin unchanged. Every other condition reports it cacheable. + let no_cache_control = edgezero_core::http::HeaderMap::new(); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + true, + StatusCode::OK, + "text/html", + &no_cache_control + ), + Some(C2BypassReason::CookieForwarded), + "cookie-personalized HTML must not become a shared template" + ); + } + #[test] fn an_origin_set_cookie_is_never_cached() { let with_cookie = headers(&[ @@ -4872,6 +4992,7 @@ mod tests { c2_bypass_reason( AssemblyMode::Esi, false, + false, StatusCode::OK, "text/html", &with_cookie @@ -4893,7 +5014,14 @@ mod tests { ] { let map = headers(&[(header::CACHE_CONTROL, directive)]); assert_eq!( - c2_bypass_reason(AssemblyMode::Esi, false, StatusCode::OK, "text/html", &map), + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map + ), Some(C2BypassReason::OriginNotShareable), "`{directive}` should disqualify the response" ); @@ -4909,6 +5037,7 @@ mod tests { c2_bypass_reason( AssemblyMode::Esi, false, + false, StatusCode::FORBIDDEN, "text/html", &shareable() @@ -4925,6 +5054,7 @@ mod tests { c2_bypass_reason( AssemblyMode::Esi, false, + false, StatusCode::OK, content_type, &shareable() @@ -4948,6 +5078,7 @@ mod tests { c2_bypass_reason( AssemblyMode::Esi, true, + false, StatusCode::FORBIDDEN, "application/json", &map From fbff408e8d83125cbb9459d72147287e123bb0d2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 09:10:20 +0530 Subject: [PATCH 185/395] Keep request-scoped diagnostics out of the shared template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the third finding from the code review. The head seam still injected request-scoped content under the shared modes, so request-neutrality was asserted for adSlots alone. Audited the seam. Of the two remaining injectors, integration head_inserts is clean: all three implementations take the context parameter unused, so their output depends on configuration and not on the request. GPT diagnostics is not clean — it is activated by a cookie or query parameter and is documented as an immutable request-scoped decision. It does not leak today, but only by coincidence. requires_private_no_store is a strict superset of the conditions under which either script is emitted, and the resulting private/no-store stamp lands before the C2 gate reads response headers, so the gate refuses. Two independent conditions that happen to align, with nothing enforcing the relationship and no test covering it. Fixed on both sides. The processor now receives no diagnostics decision under the shared modes, so the guarantee is explicit rather than emergent. And a test enumerates every combination of the decision's three fields and asserts that anything which injects also requires the stamp — so if a future change emits a script without requiring private/no-store, it fails there rather than silently in a cached template. Keeping both is deliberate: the gate is the guarantee, the invariant test is the backstop if the gate is ever removed or bypassed. Verified: fmt, all six clippy targets, all four adapter suites, 1855 core tests. --- .../src/integrations/gpt_diagnostics.rs | 60 +++++++++++++++++++ crates/trusted-server-core/src/publisher.rs | 15 ++++- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 068c43cc0..bef234d25 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -114,6 +114,66 @@ impl GptDiagnosticsRequestDecision { } } +#[cfg(test)] +mod head_seam_invariant_tests { + use super::*; + + /// Every combination of the three fields the decision carries. + fn all_decisions() -> Vec { + let mut out = Vec::new(); + for active in [false, true] { + for clean in [None, Some("/clean".to_string())] { + for cookie_action in [ + GptDiagnosticsCookieAction::None, + GptDiagnosticsCookieAction::SetSession, + GptDiagnosticsCookieAction::ClearSession, + ] { + out.push(GptDiagnosticsRequestDecision { + active, + clean_browser_path_and_query: clean.clone(), + cookie_action, + }); + } + } + } + out + } + + #[test] + fn requires_private_no_store_is_a_superset_of_injection() { + // Load-bearing relationship, not an incidental one. Whenever this decision + // injects anything into ``, the response must also be stamped + // `private, no-store` — which is what keeps request-scoped diagnostics out + // of a shared cache if the explicit assembly-mode gate in + // `create_html_stream_processor` is ever removed or bypassed. + // + // If a future change makes a script emit without also requiring the stamp, + // this fails here rather than silently in a cached template. + for decision in all_decisions() { + let injects = + decision.bootstrap_script().is_some() || decision.module_script_tag().is_some(); + if injects { + assert!( + decision.requires_private_no_store(), + "decision injects into but does not require private/no-store: \ + {decision:?}" + ); + } + } + } + + #[test] + fn a_default_decision_injects_nothing() { + let decision = GptDiagnosticsRequestDecision::default(); + assert_eq!(decision.bootstrap_script(), None); + assert_eq!(decision.module_script_tag(), None); + assert!( + !decision.requires_private_no_store(), + "an inert decision should not force the response private" + ); + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum QueryDirective { Absent, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index b8a035e2c..8fbe70f05 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1021,9 +1021,22 @@ fn create_html_stream_processor( .unwrap_or_default(); let body_close = body_close_injection(assembly_mode, params.ad_slots_script.is_some()); + // Diagnostics is request-scoped — activated by a cookie or query parameter — so + // it must not reach a shared template. It does not leak today, but only by + // coincidence: `requires_private_no_store()` is a strict superset of the + // conditions under which a script is emitted, and the resulting `private, + // no-store` stamp lands before the C2 gate reads response headers, so the gate + // refuses. That is two independent conditions happening to align. Gate it here + // instead, so the guarantee does not depend on a relationship nothing enforces. + // `gpt_diagnostics_superset_of_injection` locks the coincidence as a backstop. + let gpt_diagnostics = match assembly_mode { + AssemblyMode::Inline => params.gpt_diagnostics, + AssemblyMode::ClientFill | AssemblyMode::Esi => None, + }; + let config = config .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(params.gpt_diagnostics) + .with_gpt_diagnostics(gpt_diagnostics) .with_body_close(body_close); Ok(create_html_processor(config)) From fe88b772a6d99d90a6720726ad15bb58a70dbe47 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 09:11:51 +0530 Subject: [PATCH 186/395] Record the Task 3 code review and adopt its gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three HIGH findings, all closed in the preceding two commits. Recorded with the reasoning rather than as a list, because two of them were holes in the plan's own checklist and not merely in the implementation. The cookie gap is the clearest case: the implementation matched Task 3 Step 3's checklist exactly and still had the hole, because the checklist itself omitted the forwarded client Cookie that §4 of the design doc names. Also records what the review says about the tests. All three findings were in code the existing tests covered and passed, because those tests exercise the pure decision functions with hand-built inputs and never the rendered head or body-close bytes. That is still true — no test renders a full document through create_html_processor and compares two requests byte-for-byte, which is what the plan's Task 3 Step 2 actually requires and the most valuable test still missing. Adopts the reviewer's gate: no Task 3 Step 4 and no exposure of AssemblyMode to test or staging traffic until that test exists. The three fixes close the known holes; the test is what would catch the next one. Also notes the audit result for integration head_inserts, which is clean — all three implementations ignore the request context — so the neutrality gap was specific to diagnostics rather than general to the seam. --- .../2026-08-08-1009-measurement-findings.md | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md index 0b6a70f67..2d92512f9 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -283,6 +283,73 @@ spec flags as needing an audit and which that audit has not yet covered. Until i treat request-neutrality as asserted for one element rather than established for the template. +## Code review of the Task 3 commits — three HIGH findings, all closed + +**Date:** 2026-08-11. An independent review of the four implementation commits found +three HIGH issues. The default `Inline` path was verified unchanged byte-for-byte, so +none was a live regression — but all three were invariants this branch exists to +establish and none was enforced or tested. + +### 1. The auction dispatched under shared modes with nothing to consume it + +`assembly_mode` was computed _after_ the dispatch decision, so flipping to `client_fill` +or `esi` would still have sent real SSP bid requests, held the response for the full +auction budget, and discarded the result — because both injection seams now return +nothing — with no error, no warning and no log. + +Exactly the silent-waste signature §5 of the design doc is about, reached by an +incomplete feature flag rather than by removing the hold. Fixed by hoisting +`assembly_mode` above the dispatch and gating on `root_auction_is_useful`. + +The test derives the invariant rather than asserting per-variant: a root auction is +useful exactly when a seam will consume its result. A new mode cannot make the dispatch +gate and the injection decisions disagree without failing it. + +### 2. The C2 gate ignored the forwarded client `Cookie` + +TS forwards client cookies to origin unchanged — there is no `Cookie` strip on the +publisher path. So a response can be cookie-personalized while carrying no `Set-Cookie` +itself (session established earlier), no `Cache-Control` at all, status 200, HTML — and +every condition in the gate reported it cacheable. + +§4 of the design doc names this. The plan's own Task 3 Step 3 checklist missed it, so +the implementation matching the checklist exactly still had the hole. Now disqualifying +until an origin `Vary` covering `Cookie` is verified. + +### 3. Request-neutrality was asserted for one element, not the seam + +The head seam still injected integration `head_inserts` and the GPT-diagnostics +bootstrap unconditionally. + +Audited both. **`head_inserts` is clean** — all three implementations (datadome, didomi, +gpt) take the context parameter unused, so output depends on configuration, not the +request. **GPT diagnostics is not** — cookie- or query-activated, and documented as an +immutable request-scoped decision. + +It did not leak, but only by coincidence: `requires_private_no_store()` is a strict +superset of the conditions under which either script is emitted, and that stamp lands +before the C2 gate reads response headers, so the gate refused. Two independent +conditions that happened to align, with nothing enforcing the relationship. + +Fixed on both sides — the processor receives no diagnostics decision under shared modes, +**and** a test enumerates every combination of the decision's three fields asserting that +anything which injects also requires the stamp. The gate is the guarantee; the invariant +test is the backstop if the gate is ever removed. + +### What this says about the tests that existed + +All three findings were in code the existing tests covered — and passed. The tests +exercised the pure decision functions with hand-built inputs and never the rendered +``/`` bytes. That is still true: **no test renders a full document through +`create_html_processor` and compares two requests byte-for-byte.** The plan's Task 3 +Step 2 requires exactly that, and it remains the most valuable missing test. + +### Reviewer's gate, adopted + +Do not proceed to Task 3 Step 4 (actual C2 read/write) or expose `AssemblyMode` to any +test or staging traffic until the full-document byte-identity test exists. The three +fixes above close the known holes; that test is what would catch the next one. + ## Step B — consumers of TS's own response headers Not yet run. From 6e6b21a831c1fca3d55f4104e24cc6315934db0c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 09:23:59 +0530 Subject: [PATCH 187/395] Render whole documents and compare bytes, and prove the test has teeth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the gate the review left open, and the one the plan's Task 3 Step 2 actually asked for. Every other test in this area exercises the decision functions with hand-built inputs. That is how three HIGH review findings sat in covered, passing code: the decisions were individually right, and nothing checked what composing them renders. These tests build the config exactly as create_html_stream_processor does — same three decisions, same order — render a document through create_html_processor, and compare bytes across every combination of ad-stack gating, diagnostics activation, and bid availability. Extracted template_gpt_diagnostics so all three decisions are named functions the test can compose, rather than one of them being an inline match the test would have to duplicate. Duplicating it would have made the test agree with itself instead of with production. Mutation-tested both gates rather than trusting that passing tests mean anything. Reverting the diagnostics gate fails two of the three; reverting the head-seam gate fails the same two; the inline control passes in both cases. So the tests detect each gate independently and can still tell varying from non-varying output. Three tests rather than one, because byte-identity alone is satisfiable by rendering the same wrong thing every time. The second asserts the specific request-scoped markers that must be absent, and the third asserts inline still varies — if that one ever passes trivially, the harness is not rendering what it claims to. Adds a cfg(test) constructor for an active diagnostics decision, since the fields are private and built from a cookie or query parameter, with no other way to obtain one across a module boundary. Verified: fmt, all six clippy targets, all four adapter suites, 1858 core tests. --- .../src/integrations/gpt_diagnostics.rs | 17 ++ crates/trusted-server-core/src/publisher.rs | 210 ++++++++++++++++-- 2 files changed, 213 insertions(+), 14 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index bef234d25..3956fd682 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -114,6 +114,23 @@ impl GptDiagnosticsRequestDecision { } } +impl GptDiagnosticsRequestDecision { + /// An active decision, for tests in other modules that need one. + /// + /// The fields are private and built by `prepare_request` from a cookie or query + /// parameter; there is no other way to obtain an active decision across a module + /// boundary. + #[cfg(test)] + #[must_use] + pub(crate) fn active_for_tests() -> Self { + Self { + active: true, + clean_browser_path_and_query: None, + cookie_action: GptDiagnosticsCookieAction::None, + } + } +} + #[cfg(test)] mod head_seam_invariant_tests { use super::*; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8fbe70f05..ead7e4fb6 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -949,6 +949,29 @@ struct HtmlStreamProcessorParams<'a> { gpt_diagnostics: Option, } +/// The diagnostics decision the template may carry. +/// +/// Diagnostics is request-scoped — activated by a cookie or query parameter, and +/// documented as an immutable per-request decision — so it must not reach a shared +/// template. +/// +/// It does not leak today even without this gate, but only by coincidence: +/// `requires_private_no_store()` is a strict superset of the conditions under which +/// a script is emitted, and that stamp lands before the C2 gate reads response +/// headers, so the gate refuses. Two independent conditions that happen to align, +/// with nothing enforcing the relationship. This makes the guarantee explicit; +/// `requires_private_no_store_is_a_superset_of_injection` keeps the coincidence as a +/// backstop if this gate is ever removed. +pub(crate) fn template_gpt_diagnostics( + mode: AssemblyMode, + decision: Option, +) -> Option { + match mode { + AssemblyMode::Inline => decision, + AssemblyMode::ClientFill | AssemblyMode::Esi => None, + } +} + /// Whether a root-level auction has any consumer under this assembly mode. /// /// Only [`AssemblyMode::Inline`] injects the auction result into the root document. @@ -1021,18 +1044,7 @@ fn create_html_stream_processor( .unwrap_or_default(); let body_close = body_close_injection(assembly_mode, params.ad_slots_script.is_some()); - // Diagnostics is request-scoped — activated by a cookie or query parameter — so - // it must not reach a shared template. It does not leak today, but only by - // coincidence: `requires_private_no_store()` is a strict superset of the - // conditions under which a script is emitted, and the resulting `private, - // no-store` stamp lands before the C2 gate reads response headers, so the gate - // refuses. That is two independent conditions happening to align. Gate it here - // instead, so the guarantee does not depend on a relationship nothing enforces. - // `gpt_diagnostics_superset_of_injection` locks the coincidence as a backstop. - let gpt_diagnostics = match assembly_mode { - AssemblyMode::Inline => params.gpt_diagnostics, - AssemblyMode::ClientFill | AssemblyMode::Esi => None, - }; + let gpt_diagnostics = template_gpt_diagnostics(assembly_mode, params.gpt_diagnostics); let config = config .with_ad_state(params.ad_slots_script, params.ad_bids_state) @@ -4792,6 +4804,176 @@ mod tests { .expect("should proxy publisher request") } + mod rendered_template_identity_tests { + //! The gate the plan's Task 3 Step 2 actually asks for. + //! + //! Every other test in this area exercises the decision functions with + //! hand-built inputs. That is how three HIGH review findings sat in covered, + //! passing code: the decisions were right and nothing checked what the + //! composition of them *renders*. + //! + //! These tests render whole documents through `create_html_processor`, + //! composing the same three decisions `create_html_stream_processor` uses, + //! and compare bytes. A future request-dependent injection added at either + //! seam fails here even if every decision function is left untouched. + + use super::template_neutrality_tests::{settings_with_slots, slot}; + use super::*; + use crate::creative_opportunities::AssemblyMode; + use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; + use crate::integrations::IntegrationRegistry; + use crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision; + + const DOCUMENT: &[u8] = + b"t

content

"; + + /// One request's worth of variation. Everything here is request-scoped and + /// must not reach a shared template. + #[derive(Debug, Clone, Copy)] + struct RequestShape { + /// Folds in consent, bot classification, prefetch and the kill switch. + ad_stack_ran: bool, + /// Cookie- or query-activated. + diagnostics_active: bool, + /// A resolved auction, present only when one was dispatched. + bids_available: bool, + } + + /// Build the config exactly as `create_html_stream_processor` does, so a + /// drift between a decision and its use is caught rather than hidden. + fn render(mode: AssemblyMode, shape: RequestShape) -> String { + let settings = settings_with_slots(); + let slots = [slot()]; + + let ad_slots_script = + template_ad_slots_script(mode, shape.ad_stack_ran, &settings, &slots, "/"); + let body_close = body_close_injection(mode, ad_slots_script.is_some()); + let gpt_diagnostics = template_gpt_diagnostics( + mode, + shape + .diagnostics_active + .then(GptDiagnosticsRequestDecision::active_for_tests), + ); + + let ad_bids_state = + std::sync::Arc::new(std::sync::Mutex::new(shape.bids_available.then(|| { + r#""#.to_string() + }))); + + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script, + ad_bids_state, + max_buffered_body_bytes: 16 * 1024 * 1024, + gpt_diagnostics, + body_close, + }; + + let mut processor = create_html_processor(config); + let out = processor + .process_chunk(DOCUMENT, true) + .expect("should process the document"); + String::from_utf8(out).expect("output should be utf8") + } + + fn every_shape() -> Vec { + let mut shapes = Vec::new(); + for ad_stack_ran in [false, true] { + for diagnostics_active in [false, true] { + for bids_available in [false, true] { + shapes.push(RequestShape { + ad_stack_ran, + diagnostics_active, + bids_available, + }); + } + } + } + shapes + } + + #[test] + fn shared_modes_render_byte_identical_documents_for_every_request_shape() { + for mode in [AssemblyMode::ClientFill, AssemblyMode::Esi] { + let shapes = every_shape(); + let baseline = render(mode, shapes[0]); + + for shape in &shapes[1..] { + let rendered = render(mode, *shape); + assert_eq!( + rendered, baseline, + "{mode:?}: rendered template differs for {shape:?}. A shared \ + template that varies by request freezes the first-filling \ + request's decision for every later reader." + ); + } + } + } + + #[test] + fn shared_mode_templates_contain_no_request_scoped_markers() { + // Byte-identity alone would be satisfied by rendering the same wrong + // thing every time, so also assert the specific things that must be + // absent. + for mode in [AssemblyMode::ClientFill, AssemblyMode::Esi] { + let rendered = render( + mode, + RequestShape { + ad_stack_ran: true, + diagnostics_active: true, + bids_available: true, + }, + ); + for forbidden in [ + ".adSlots", + ".bids=", + "__tsjs_gpt_diagnostics_active", + "history.replaceState", + ] { + assert!( + !rendered.contains(forbidden), + "{mode:?}: template contains request-scoped `{forbidden}`:\n{rendered}" + ); + } + } + } + + #[test] + fn inline_still_varies_by_request_as_it_must() { + // The shared-mode assertions would also pass if rendering were broken + // everywhere. Inline responses are per-navigation and never shared, so + // they *should* differ — this proves the test can tell the difference. + let with_ads = render( + AssemblyMode::Inline, + RequestShape { + ad_stack_ran: true, + diagnostics_active: false, + bids_available: true, + }, + ); + let without = render( + AssemblyMode::Inline, + RequestShape { + ad_stack_ran: false, + diagnostics_active: false, + bids_available: false, + }, + ); + assert_ne!( + with_ads, without, + "inline must still vary by request; if it does not, this harness is \ + not rendering what it claims to" + ); + assert!( + with_ads.contains(".adSlots"), + "inline with a matched slot should carry adSlots" + ); + } + } + mod root_auction_gate_tests { //! Guards the silent-waste failure mode: dispatching an auction whose result //! nothing will consume. Under the shared modes both injection seams emit @@ -5115,7 +5297,7 @@ mod tests { AssemblyMode, CreativeOpportunityFormat, CreativeOpportunitySlot, }; - fn slot() -> CreativeOpportunitySlot { + pub(super) fn slot() -> CreativeOpportunitySlot { CreativeOpportunitySlot { id: "atf".to_string(), gam_unit_path: Some("/99999/example/home".to_string()), @@ -5134,7 +5316,7 @@ mod tests { } } - fn settings_with_slots() -> Settings { + pub(super) fn settings_with_slots() -> Settings { let mut settings = crate::test_support::tests::create_test_settings(); // Construct the section rather than mutating it if present: the shared // fixture does not carry `[creative_opportunities]`, and an `if let From 04a3196520c80d1b2f5075fc690164dc68530899 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 09:50:22 +0530 Subject: [PATCH 188/395] Add the C2 template cache trait, key and null object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First half of Task 3 Step 4, in portable core. No Fastly implementation yet and no call site, so nothing changes behaviour — this is the shape the adapter will fill in. Follows the PlatformKvStore pattern the repo already uses four times for a Fastly-only capability behind a portable trait with a null object. The null object reports Unsupported rather than erroring, so the shared assembly modes degrade to transforming per request on Cloudflare, Axum and Spin instead of failing there. The modes stay portable; only the caching does not. The key is where the correctness risks live, and it carries the four the design investigation surfaced. Assembly mode, because the client-fill and ESI arms emit different bytes and would otherwise poison each other's entries. Content encoding, because the pipeline pairs input encoding to output encoding, so serving brotli bytes to a client that asked for gzip is a broken response. Host and scheme, because both reach IntegrationHtmlContext and drive URL rewriting. And a schema version, so a deploy that changes the transform reads a miss rather than assembling against markers that moved. Vary values are carried as the origin listed them rather than as a fixed list, because the origin is authoritative and a hard-coded list would drift silently when the origin's changes. Step A already measured four Next-specific headers this branch did not anticipate. Fields are length-prefixed rather than delimiter-joined. A delimiter is ambiguous when a value can contain it, and two distinct keys colliding here means one visitor's template served to another. There is a test for exactly that collision. Metadata is a small envelope rather than stored origin headers. The publisher path forces private/no-store and strips validators after the send, so replaying a stored origin header would fight it; rebuilding every header on a hit means no origin header is ever replayed and the Set-Cookie privacy net stays trivially safe. Malformed metadata decodes to a miss rather than a partial read. Eight tests. The one worth naming asserts every field changes the key — a field that does not is a cross-serving bug, and that property is easy to break by adding a field and forgetting to hash it. Verified: fmt, all six clippy targets, all four adapter suites, 1866 core tests. --- .../trusted-server-core/src/platform/mod.rs | 5 + .../src/platform/template_cache.rs | 493 ++++++++++++++++++ 2 files changed, 498 insertions(+) create mode 100644 crates/trusted-server-core/src/platform/template_cache.rs diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 287f1accf..7cab20d29 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -36,6 +36,7 @@ mod error; mod http; mod image_optimizer; mod kv; +pub mod template_cache; #[cfg(test)] pub(crate) mod test_support; mod traits; @@ -52,6 +53,10 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; +pub use template_cache::{ + PlatformTemplateCache, TEMPLATE_SCHEMA_VERSION, TemplateCacheError, TemplateCacheKey, + TemplateCacheMiss, TemplateEntry, TemplateMetadata, UnavailableTemplateCache, +}; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs new file mode 100644 index 000000000..c89926484 --- /dev/null +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -0,0 +1,493 @@ +//! The shared transformed-template cache (C2) for the #1009 ESI validation spike. +//! +//! Three caches are in play and conflating them is what produced the original wrong +//! conclusion in the design doc, so this module names which one it is: +//! +//! | Cache | Contents | Owner | +//! | ----- | --------------------------------- | ------------------------------ | +//! | C1 | raw origin bytes | Fastly read-through. Not this. | +//! | C2 | post-`lol_html`, pre-assembly | **This module.** | +//! | C3 | final per-user assembled response | **Must never exist.** | +//! +//! C2 holds a *shared template*: no per-user bytes, and no decisions that depend on +//! the request. What may and may not live in it is +//! [§6.7 of the design doc](../../../../docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md), +//! and the invariant is enforced by the rendered-document byte-identity tests in +//! `publisher`. +//! +//! Spike-only. Remove with the spike. + +use core::fmt; + +use crate::creative_opportunities::AssemblyMode; + +/// Version of the transform that produced a cached template. +/// +/// Bump on **any** change to what the transform emits. Without it a deploy reads +/// yesterday's template shape and assembles against markers that moved, which fails +/// as a rendering bug far from its cause rather than as a cache miss. +pub const TEMPLATE_SCHEMA_VERSION: u32 = 1; + +/// Inputs that select one cached template. +/// +/// Every field changes the emitted bytes for the same URL. A signal that changes the +/// bytes and is **not** here produces cross-served templates; a signal that is +/// per-user does not belong here at all — it belongs out of the template entirely. +/// That distinction is the whole design: the key holds per-*variant* signals, and +/// per-*user* signals are excluded from the template rather than keyed on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateCacheKey { + /// Full request URL, stated explicitly rather than inherited from an ambient + /// request, so the key cannot silently depend on what the caller happened to + /// mutate first. + pub url: String, + /// Host and scheme. The post-processed output is host-dependent by construction: + /// both reach `IntegrationHtmlContext` and drive URL rewriting. + pub request_host: String, + /// See [`Self::request_host`]. + pub request_scheme: String, + /// A2 and A3 emit different template bytes. Without this they poison each + /// other's entries. + pub assembly_mode: AssemblyMode, + /// Values of the request headers the **origin** declares it varies on, in the + /// order the origin listed them. Not a fixed list: the origin is authoritative, + /// and hard-coding one here would silently drift when the origin's changes. + pub vary_values: Vec<(String, String)>, + /// The negotiated content encoding of the stored bytes. + /// + /// The streaming pipeline pairs input encoding to the same output encoding, so + /// the transformed bytes inherit whatever the origin chose from the client's + /// `Accept-Encoding`. Serving brotli bytes to a client that asked for gzip is a + /// broken response, so this is part of the key rather than of the payload. + pub content_encoding: String, + /// Identifies the enabled integration set and the tsjs bundle. Both change the + /// injected markup for the same URL. + pub integration_fingerprint: String, + /// See [`TEMPLATE_SCHEMA_VERSION`]. + pub schema_version: u32, +} + +impl TemplateCacheKey { + /// Render the key as the opaque byte string the platform cache is keyed on. + /// + /// Fields are length-prefixed rather than delimiter-joined. A delimiter is + /// ambiguous when a value can contain it — a URL with a `|`, or a `Vary` value + /// with one — and two distinct keys colliding here means one visitor's template + /// served to another. Length prefixes make that unrepresentable. + #[must_use] + pub fn to_cache_key(&self) -> String { + let mut out = String::new(); + let mut push = |part: &str| { + out.push_str(&part.len().to_string()); + out.push(':'); + out.push_str(part); + }; + + push("ts-c2"); + push(&self.schema_version.to_string()); + push(&format!("{:?}", self.assembly_mode)); + push(&self.request_scheme); + push(&self.request_host); + push(&self.url); + push(&self.content_encoding); + push(&self.integration_fingerprint); + + push(&self.vary_values.len().to_string()); + for (name, value) in &self.vary_values { + push(&name.to_ascii_lowercase()); + push(value); + } + + out + } + + /// Surrogate keys to attach at insert, for purge-based rollback. + /// + /// `ts-template` purges every template at once, which is the rollback lever. + /// The per-URL key allows targeted invalidation. Both are needed: the broad one + /// for an incident, the narrow one for ordinary invalidation. + #[must_use] + pub fn surrogate_keys(&self) -> Vec { + vec![ + "ts-template".to_string(), + format!("ts-template-{}", surrogate_safe(&self.url)), + ] + } +} + +/// Reduce a URL to characters valid in a Fastly surrogate key. +/// +/// Surrogate keys are space-delimited, so any whitespace would split one key into +/// several and purge more than intended. +fn surrogate_safe(url: &str) -> String { + url.chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' || c == '_' { + c + } else { + '_' + } + }) + .collect() +} + +/// Metadata stored alongside the template bytes. +/// +/// `cache::core` carries **no HTTP semantics** — status, headers, encoding and +/// revalidation are all the caller's. Rather than storing origin headers and +/// replaying them, store only what is needed to rebuild a response from scratch. +/// +/// That choice is deliberate and load-bearing: the publisher path forces +/// `private, no-store` and strips validators *after* the origin send, so replaying a +/// stored origin header would fight it. Rebuilding every header on a hit means no +/// origin header is ever replayed and the `Set-Cookie` privacy net stays trivially +/// safe. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateMetadata { + /// Encoding of the stored bytes. Also in the key; stored so a reader need not + /// re-derive it. + pub content_encoding: String, + /// Content type to rebuild the response with. + pub content_type: String, + /// Schema version the bytes were produced under. Checked on read: a mismatch is + /// a miss, not an error, so a rollback to an older binary degrades to + /// re-transforming rather than misassembling. + pub schema_version: u32, +} + +impl TemplateMetadata { + /// Serialize for `user_metadata`. Deliberately a tiny hand-rolled format rather + /// than JSON — one allocation, no dependency, and a parse failure is + /// unambiguous. + #[must_use] + pub fn encode(&self) -> Vec { + format!( + "v={}\nce={}\nct={}", + self.schema_version, self.content_encoding, self.content_type + ) + .into_bytes() + } + + /// Parse `user_metadata`. Returns `None` on anything unexpected, which callers + /// must treat as a cache miss. + #[must_use] + pub fn decode(raw: &[u8]) -> Option { + let text = core::str::from_utf8(raw).ok()?; + let mut schema_version = None; + let mut content_encoding = None; + let mut content_type = None; + for line in text.lines() { + let (key, value) = line.split_once('=')?; + match key { + "v" => schema_version = Some(value.parse().ok()?), + "ce" => content_encoding = Some(value.to_string()), + "ct" => content_type = Some(value.to_string()), + _ => return None, + } + } + Some(Self { + schema_version: schema_version?, + content_encoding: content_encoding?, + content_type: content_type?, + }) + } +} + +/// Why a template read did not produce usable bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum TemplateCacheMiss { + /// No entry for this key. + #[display("no cached template for this key")] + NotFound, + /// Found, but produced by a different transform version. + #[display("cached template has a different schema version")] + SchemaMismatch, + /// Found, but its metadata could not be parsed. + #[display("cached template metadata is unreadable")] + UnreadableMetadata, + /// This platform has no template cache. + #[display("no template cache on this platform")] + Unsupported, +} + +impl core::error::Error for TemplateCacheMiss {} + +/// Errors a template cache write can produce. +#[derive(Debug, derive_more::Display)] +pub enum TemplateCacheError { + /// This platform has no template cache. + #[display("no template cache on this platform")] + Unsupported, + /// The platform rejected the operation. + #[display("template cache backend error: {message}")] + Backend { + /// What the backend reported. + message: String, + }, +} + +impl core::error::Error for TemplateCacheError {} + +impl fmt::Debug for dyn PlatformTemplateCache { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformTemplateCache") + } +} + +/// A platform's shared-template cache. +/// +/// Only the Fastly adapter implements this; every other adapter uses +/// [`UnavailableTemplateCache`], which reports [`TemplateCacheMiss::Unsupported`] so +/// the caller transforms every time rather than failing. +#[async_trait::async_trait(?Send)] +pub trait PlatformTemplateCache { + /// Read a template. `Err` is a miss, not a failure — every variant means + /// "transform it yourself". + async fn get(&self, key: &TemplateCacheKey) -> Result; + + /// Store a template. + /// + /// Callers must not call this without having consulted the C2 eligibility gate + /// first: this method stores what it is given and cannot tell a shared template + /// from a per-user one. + async fn put( + &self, + key: &TemplateCacheKey, + metadata: &TemplateMetadata, + body: Vec, + ) -> Result<(), TemplateCacheError>; + + /// Purge every stored template. The rollback lever. + async fn purge_all(&self) -> Result<(), TemplateCacheError>; +} + +/// A template read from the cache. +pub struct TemplateEntry { + /// Metadata stored at insert. + pub metadata: TemplateMetadata, + /// The transformed template bytes. + pub body: Vec, +} + +/// The null object, used by every adapter without a template cache. +/// +/// Reporting [`TemplateCacheMiss::Unsupported`] rather than erroring means the +/// shared assembly modes degrade to transforming per request on Cloudflare, Axum and +/// Spin instead of failing — the modes stay portable, only the caching is not. +pub struct UnavailableTemplateCache; + +#[async_trait::async_trait(?Send)] +impl PlatformTemplateCache for UnavailableTemplateCache { + async fn get(&self, _key: &TemplateCacheKey) -> Result { + Err(TemplateCacheMiss::Unsupported) + } + + async fn put( + &self, + _key: &TemplateCacheKey, + _metadata: &TemplateMetadata, + _body: Vec, + ) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } + + async fn purge_all(&self) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key() -> TemplateCacheKey { + TemplateCacheKey { + url: "https://example.com/news/article".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + assembly_mode: AssemblyMode::Esi, + vary_values: vec![("rsc".to_string(), "1".to_string())], + content_encoding: "gzip".to_string(), + integration_fingerprint: "abc123".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + } + } + + /// Every field must change the key. A field that does not is a cross-serving + /// bug: two requests needing different templates would share one entry. + #[test] + fn every_field_changes_the_key() { + let base = key().to_cache_key(); + + let mut mode = key(); + mode.assembly_mode = AssemblyMode::ClientFill; + assert_ne!( + mode.to_cache_key(), + base, + "assembly mode must change the key" + ); + + let mut url = key(); + url.url = "https://example.com/other".to_string(); + assert_ne!(url.to_cache_key(), base, "url must change the key"); + + let mut host = key(); + host.request_host = "other.example.com".to_string(); + assert_ne!(host.to_cache_key(), base, "host must change the key"); + + let mut scheme = key(); + scheme.request_scheme = "http".to_string(); + assert_ne!(scheme.to_cache_key(), base, "scheme must change the key"); + + let mut encoding = key(); + encoding.content_encoding = "br".to_string(); + assert_ne!( + encoding.to_cache_key(), + base, + "content encoding must change the key; serving brotli to a gzip client \ + is a broken response" + ); + + let mut fingerprint = key(); + fingerprint.integration_fingerprint = "def456".to_string(); + assert_ne!( + fingerprint.to_cache_key(), + base, + "integration fingerprint must change the key" + ); + + let mut schema = key(); + schema.schema_version = TEMPLATE_SCHEMA_VERSION + 1; + assert_ne!( + schema.to_cache_key(), + base, + "schema version must change the key" + ); + + let mut vary = key(); + vary.vary_values = vec![("rsc".to_string(), "0".to_string())]; + assert_ne!(vary.to_cache_key(), base, "vary values must change the key"); + } + + /// The reason for length prefixes rather than a delimiter. + #[test] + fn values_containing_delimiters_cannot_collide() { + let mut a = key(); + a.request_host = "a".to_string(); + a.url = "b:c".to_string(); + + let mut b = key(); + b.request_host = "a:b".to_string(); + b.url = "c".to_string(); + + assert_ne!( + a.to_cache_key(), + b.to_cache_key(), + "field values containing the delimiter must not produce the same key; a \ + collision here serves one visitor's template to another" + ); + } + + #[test] + fn vary_header_names_are_matched_case_insensitively() { + let mut upper = key(); + upper.vary_values = vec![("RSC".to_string(), "1".to_string())]; + assert_eq!( + upper.to_cache_key(), + key().to_cache_key(), + "header names are case-insensitive, so casing must not split the cache" + ); + } + + #[test] + fn vary_values_are_order_sensitive() { + // The origin lists them in a fixed order and the caller preserves it, so a + // differing order means differing inputs rather than the same request. + let mut a = key(); + a.vary_values = vec![ + ("rsc".to_string(), "1".to_string()), + ("accept-encoding".to_string(), "gzip".to_string()), + ]; + let mut b = key(); + b.vary_values = vec![ + ("accept-encoding".to_string(), "gzip".to_string()), + ("rsc".to_string(), "1".to_string()), + ]; + assert_ne!(a.to_cache_key(), b.to_cache_key()); + } + + #[test] + fn surrogate_keys_carry_a_global_and_a_per_url_lever() { + let keys = key().surrogate_keys(); + assert!( + keys.contains(&"ts-template".to_string()), + "a global purge lever is what makes rollback possible" + ); + assert_eq!(keys.len(), 2, "global plus per-URL"); + assert!( + !keys[1].contains(char::is_whitespace), + "surrogate keys are space-delimited; whitespace would purge more than \ + intended, got {:?}", + keys[1] + ); + assert!( + !keys[1].contains('/') && !keys[1].contains(':'), + "URL punctuation must be reduced, got {:?}", + keys[1] + ); + } + + #[test] + fn metadata_round_trips() { + let metadata = TemplateMetadata { + content_encoding: "gzip".to_string(), + content_type: "text/html; charset=utf-8".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + }; + let decoded = + TemplateMetadata::decode(&metadata.encode()).expect("should decode what it encoded"); + assert_eq!(decoded, metadata); + } + + #[test] + fn unparseable_metadata_is_a_miss_not_a_panic() { + for raw in [ + &b"not-key-value"[..], + &b"v=notanumber\nce=gzip\nct=text/html"[..], + &b"v=1\nce=gzip"[..], + &b"v=1\nce=gzip\nct=text/html\nunexpected=1"[..], + &[0xff, 0xfe][..], + ] { + assert_eq!( + TemplateMetadata::decode(raw), + None, + "malformed metadata must be a miss, not a partial read: {raw:?}" + ); + } + } + + #[tokio::test] + async fn the_null_object_reports_unsupported_rather_than_failing() { + // Degrading to per-request transformation keeps the shared modes portable on + // adapters with no cache; erroring would make them Fastly-only outright. + let cache = UnavailableTemplateCache; + assert_eq!( + cache.get(&key()).await.err(), + Some(TemplateCacheMiss::Unsupported) + ); + assert!(matches!( + cache + .put( + &key(), + &TemplateMetadata { + content_encoding: "identity".to_string(), + content_type: "text/html".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + }, + Vec::new() + ) + .await, + Err(TemplateCacheError::Unsupported) + )); + } +} From ecc6c5306f356d3ad3ed17df14d5d3d3b37a37d6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 10:20:33 +0530 Subject: [PATCH 189/395] Back the C2 template cache with Fastly Core Cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes Task 3 Step 4. The cache is constructed and reachable through RuntimeServices but has no caller yet, and the assembly mode defaults to Inline, so nothing changes behaviour. Seven tests run against real Core Cache under Viceroy, including purge. That is what the earlier probe established was possible and why provisioning a Fastly service is not on the critical path. Two ordering traps, both caught by the earlier reviews and both real here. must_insert_or_update is tested before found, because a stale entry sets both and checking found first would serve the stale bytes while never discharging the obligation, leaving concurrent waiters blocked until timeout. And get uses a plain lookup rather than a transaction, because a read that never intends to insert must not take an obligation it will not discharge. Transaction::insert takes self, so once the insert begins there is no handle left to cancel it with — a write that fails part-way cannot be retracted. Rather than write a cancel call that does not compile, or pretend the hazard is absent, the metadata carries the intended body length and get rejects a short entry as Truncated. put also refuses a length that disagrees with the body it was given, since storing that would make every subsequent read a truncation miss: a cache that silently never hits. Also treats a stale entry as a miss. Serving stale while revalidating is a real option but it is a state machine cache::core does not implement, and it is not what this spike measures. The trait is Send + Sync with ?Send futures. RuntimeServices lives in a LazyLock static so the trait object must cross threads, while the platform layer is !Send by construction and the futures never do. Wired into RuntimeServices following the kv_store pattern, but defaulted rather than required: an adapter with no template cache should degrade to transforming per request, not fail to build. That is what keeps the shared modes portable across all four adapters with only the caching being Fastly-only. Verified: fmt, all six clippy targets, all four adapter suites, 1866 core tests and 123 Fastly adapter tests. --- .../trusted-server-adapter-fastly/src/app.rs | 6 + .../trusted-server-adapter-fastly/src/main.rs | 1 + .../src/template_cache.rs | 327 ++++++++++++++++++ .../src/platform/template_cache.rs | 34 +- .../trusted-server-core/src/platform/types.rs | 36 ++ 5 files changed, 398 insertions(+), 6 deletions(-) create mode 100644 crates/trusted-server-adapter-fastly/src/template_cache.rs diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 5258d3455..c85e24670 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -256,6 +256,12 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime .config_store(Arc::new(FastlyPlatformConfigStore)) .secret_store(Arc::new(FastlyPlatformSecretStore)) .kv_store(Arc::clone(&state.default_kv_store)) + // Spike-only (#1009). Constructed unconditionally, but only read when the + // assembly mode is a shared-template one — which defaults to Inline, so this + // is inert until an operator opts in. + .template_cache(Arc::new(crate::template_cache::FastlyTemplateCache::new( + crate::template_cache::TEMPLATE_CACHE_TTL, + ))) .backend(Arc::new(FastlyPlatformBackend)) .http_client(Arc::new(FastlyPlatformHttpClient)) .geo(Arc::new(FastlyPlatformGeo)) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 39d35b198..603ffdd9b 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -34,6 +34,7 @@ mod management_api; mod middleware; mod platform; mod rate_limiter; +mod template_cache; mod tinybird; use crate::app::{EcFinalizeState, TrustedServerApp, load_settings_from_config_store}; diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs new file mode 100644 index 000000000..d037a3e6a --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -0,0 +1,327 @@ +//! Fastly Core Cache backing for the shared transformed-template cache (C2). +//! +//! Only the Fastly adapter implements this; every other adapter uses +//! `UnavailableTemplateCache`, so the shared assembly modes stay portable and only +//! the caching is Fastly-only. +//! +//! **Why Core Cache and not read-through caching.** Read-through with `after_send` + +//! `set_body_transform` looks like a better fit — it keeps HTTP semantics and derives +//! TTL and surrogate keys from origin headers for free. It is unreachable here: +//! Viceroy 0.17 stubs the entire HTTP Cache ABI and the SDK converts that into a +//! *send error*, so setting `after_send` makes every publisher origin fetch fail +//! under `fastly compute serve`, `cargo test-fastly` and the parity suite. It is also +//! silently dead whenever the origin request is in pass mode, and its closure bounds +//! (`Fn + Send + Sync`) are incompatible with a platform layer that is `!Send` by +//! construction. Recorded in the spike plan's Task 3 Step 4 so nobody re-proposes it. +//! +//! Spike-only. Remove with the spike. + +use fastly::cache::core::{CacheKey, Transaction}; +use std::io::Write as _; +use std::time::Duration; +use trusted_server_core::platform::{ + PlatformTemplateCache, TemplateCacheError, TemplateCacheKey, TemplateCacheMiss, TemplateEntry, + TemplateMetadata, +}; + +/// How long a cached template lives. +/// +/// Deliberately short for the spike. A short TTL bounds every failure mode in this +/// module — a poisoned template, a stale schema, a truncated write — and the only +/// cost is hit rate, which is a measurement input rather than a correctness one. +pub const TEMPLATE_CACHE_TTL: Duration = Duration::from_secs(60); + +/// Surrogate key attached to every stored template, so a single purge clears them +/// all. This is the rollback lever: without it, backing out a bad template means +/// waiting for the TTL. +const PURGE_ALL_SURROGATE_KEY: &str = "ts-template"; + +/// Fastly Core Cache implementation of the C2 template cache. +pub struct FastlyTemplateCache { + ttl: Duration, +} + +impl FastlyTemplateCache { + /// Create a cache whose entries live for `ttl`. + /// + /// Keep this short for the spike. A short TTL bounds every failure mode in this + /// module — a poisoned template, a stale schema, a bad transform — and costs + /// only hit rate. + #[must_use] + pub fn new(ttl: Duration) -> Self { + Self { ttl } + } +} + +fn backend_error(message: impl Into) -> TemplateCacheError { + TemplateCacheError::Backend { + message: message.into(), + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformTemplateCache for FastlyTemplateCache { + async fn get(&self, key: &TemplateCacheKey) -> Result { + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + // A plain lookup, not a transaction: a read that does not intend to insert + // must not take an insert obligation it will never discharge, which would + // block every other client waiting on the same key until they time out. + let found = fastly::cache::core::lookup(cache_key) + .execute() + .map_err(|_| TemplateCacheMiss::NotFound)? + .ok_or(TemplateCacheMiss::NotFound)?; + + // Stale entries are treated as a miss for the spike. Serving stale while + // revalidating is a real option, but it is a state machine `cache::core` + // does not implement for you, and it is not what this spike is measuring. + if found.is_stale() { + return Err(TemplateCacheMiss::NotFound); + } + + let metadata = TemplateMetadata::decode(&found.user_metadata()) + .ok_or(TemplateCacheMiss::UnreadableMetadata)?; + + // A schema mismatch is a miss, not an error: rolling back to an older binary + // then degrades to re-transforming rather than assembling against a template + // shape it does not understand. + if metadata.schema_version != key.schema_version { + return Err(TemplateCacheMiss::SchemaMismatch); + } + + let body = found + .to_stream() + .map_err(|_| TemplateCacheMiss::NotFound)? + .into_bytes(); + + // A write that failed part-way cannot cancel its own insert (see `put`), so + // a short entry is possible. Catch it here rather than assembling a + // truncated template into a broken page. + if body.len() as u64 != metadata.body_len { + return Err(TemplateCacheMiss::Truncated); + } + + Ok(TemplateEntry { metadata, body }) + } + + async fn put( + &self, + key: &TemplateCacheKey, + metadata: &TemplateMetadata, + body: Vec, + ) -> Result<(), TemplateCacheError> { + if metadata.body_len != body.len() as u64 { + return Err(backend_error(format!( + "metadata body_len {} does not match the {} bytes supplied; storing \ + this would make every read a truncation miss", + metadata.body_len, + body.len() + ))); + } + + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + // Transactional insert so a cold key under load transforms once rather than + // once per concurrent request. + let tx = Transaction::lookup(cache_key) + .execute() + .map_err(|e| backend_error(format!("transactional lookup failed: {e:?}")))?; + + // Order matters. A STALE entry sets *both* `found()` and + // `must_insert_or_update()`. Testing `found()` first would return early on + // the stale bytes and never discharge the obligation, leaving every + // concurrent waiter blocked until timeout. + if !tx.must_insert_or_update() { + // Someone else already inserted a fresh entry. Nothing to do, and + // nothing to discharge. + return Ok(()); + } + + // `Transaction::insert` takes `self`, so from here there is no handle left to + // cancel the insert with. A write that fails part-way therefore cannot be + // retracted — which is why `TemplateMetadata::body_len` exists and `get` + // checks it. The metadata is written before the body, so a truncated entry + // still carries the length it was supposed to have. + let surrogate_keys = key.surrogate_keys(); + let mut writer = tx + .insert(self.ttl) + .surrogate_keys(surrogate_keys.iter().map(String::as_str)) + .user_metadata(metadata.encode().into()) + .execute() + .map_err(|e| backend_error(format!("cache insert failed: {e:?}")))?; + + if let Err(e) = writer.write_all(&body) { + // Deliberately not calling `finish()`. An unfinished entry has no known + // length, and even if it is observable, `get`'s length check rejects it. + return Err(backend_error(format!("writing template body failed: {e}"))); + } + + // Required. Without it the object never completes and its length stays + // unknown, so readers see a partial or absent entry. + writer + .finish() + .map_err(|e| backend_error(format!("finishing the cached template failed: {e}")))?; + + Ok(()) + } + + async fn purge_all(&self) -> Result<(), TemplateCacheError> { + fastly::http::purge::purge_surrogate_key(PURGE_ALL_SURROGATE_KEY) + .map_err(|e| backend_error(format!("purging templates failed: {e:?}"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use trusted_server_core::creative_opportunities::AssemblyMode; + use trusted_server_core::platform::TEMPLATE_SCHEMA_VERSION; + + /// Distinct per test, so tests sharing the process cache cannot collide. + fn key(url: &str) -> TemplateCacheKey { + TemplateCacheKey { + url: url.to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + assembly_mode: AssemblyMode::Esi, + vary_values: vec![("rsc".to_string(), "1".to_string())], + content_encoding: "identity".to_string(), + integration_fingerprint: "fp".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + } + } + + fn metadata_for(body: &[u8]) -> TemplateMetadata { + TemplateMetadata { + content_encoding: "identity".to_string(), + content_type: "text/html; charset=utf-8".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: body.len() as u64, + } + } + + /// The trait is `async_trait(?Send)` and this crate has no async test runtime, + /// so drive the futures directly. + fn run(fut: impl core::future::Future) -> T { + futures::executor::block_on(fut) + } + + fn cache() -> FastlyTemplateCache { + FastlyTemplateCache::new(Duration::from_secs(60)) + } + + #[test] + fn a_stored_template_reads_back_intact() { + let cache = cache(); + let key = key("https://example.com/roundtrip"); + let body = b"template".to_vec(); + let metadata = metadata_for(&body); + + run(cache.put(&key, &metadata, body.clone())).expect("should store"); + + let entry = run(cache.get(&key)).expect("should read back"); + assert_eq!(entry.body, body, "bytes must survive the round trip"); + assert_eq!(entry.metadata, metadata, "metadata must survive too"); + } + + #[test] + fn an_absent_key_is_a_miss_not_an_error() { + let miss = + run(cache().get(&key("https://example.com/never-stored"))).expect_err("should miss"); + assert_eq!(miss, TemplateCacheMiss::NotFound); + } + + #[test] + fn a_different_assembly_mode_does_not_read_the_same_entry() { + // The arms emit different bytes. If they shared an entry, one would serve + // the other's template. + let cache = cache(); + let esi = key("https://example.com/mode-split"); + let mut client_fill = esi.clone(); + client_fill.assembly_mode = AssemblyMode::ClientFill; + + let body = b"esi-template".to_vec(); + run(cache.put(&esi, &metadata_for(&body), body)).expect("should store"); + + assert_eq!( + run(cache.get(&client_fill)).err(), + Some(TemplateCacheMiss::NotFound), + "client-fill must not read the ESI arm's template" + ); + } + + #[test] + fn a_schema_bump_reads_a_miss_rather_than_a_stale_shape() { + let cache = cache(); + let key_v1 = key("https://example.com/schema"); + let body = b"old-shape".to_vec(); + run(cache.put(&key_v1, &metadata_for(&body), body)).expect("should store"); + + // A deploy that changes the transform bumps the constant. The old entry must + // not be assembled against. + let mut key_v2 = key_v1.clone(); + key_v2.schema_version = TEMPLATE_SCHEMA_VERSION + 1; + + assert_eq!( + run(cache.get(&key_v2)).err(), + Some(TemplateCacheMiss::NotFound), + "a bumped schema changes the key, so the old entry is simply not found" + ); + } + + #[test] + fn purge_all_clears_stored_templates() { + // The rollback lever. Without this, backing out a bad template means waiting + // for the TTL. + let cache = cache(); + let key = key("https://example.com/purge"); + let body = b"template".to_vec(); + run(cache.put(&key, &metadata_for(&body), body)).expect("should store"); + run(cache.get(&key)).expect("should be present before purge"); + + run(cache.purge_all()).expect("should purge"); + + assert!( + run(cache.get(&key)).is_err(), + "purge must clear the template, or rollback is TTL-bound" + ); + } + + #[test] + fn a_second_put_on_a_fresh_entry_is_a_no_op() { + // Exercises the `must_insert_or_update` early return: a concurrent writer + // that finds a fresh entry must neither error nor overwrite. + let cache = cache(); + let key = key("https://example.com/second-put"); + let first = b"first".to_vec(); + run(cache.put(&key, &metadata_for(&first), first.clone())).expect("first put stores"); + + let second = b"second".to_vec(); + run(cache.put(&key, &metadata_for(&second), second)) + .expect("second put should be a no-op, not an error"); + + assert_eq!( + run(cache.get(&key)).expect("should read").body, + first, + "a fresh entry must not be overwritten by a racing writer" + ); + } + + #[test] + fn a_length_mismatch_is_refused_at_write_rather_than_stored() { + // Storing metadata whose length disagrees with the body would make every + // subsequent read a truncation miss — a cache that silently never hits. + // Catch it at the write instead. + let cache = cache(); + let key = key("https://example.com/length-mismatch"); + let mut metadata = metadata_for(b"12345"); + metadata.body_len = 999; + + let err = run(cache.put(&key, &metadata, b"12345".to_vec())) + .expect_err("a length mismatch must be refused"); + assert!( + matches!(err, TemplateCacheError::Backend { .. }), + "expected a backend error, got {err:?}" + ); + } +} diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index c89926484..da5ebf68f 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -153,6 +153,14 @@ pub struct TemplateMetadata { /// a miss, not an error, so a rollback to an older binary degrades to /// re-transforming rather than misassembling. pub schema_version: u32, + /// Length of the template bytes as written. + /// + /// Guards against a partially written entry. `Transaction::insert` consumes the + /// transaction, so a write that fails part-way cannot cancel the insert — there + /// is no handle left to cancel it with. Recording the intended length and + /// checking it on read makes a truncated entry a miss instead of a silently + /// short template that would assemble into a broken page. + pub body_len: u64, } impl TemplateMetadata { @@ -162,8 +170,8 @@ impl TemplateMetadata { #[must_use] pub fn encode(&self) -> Vec { format!( - "v={}\nce={}\nct={}", - self.schema_version, self.content_encoding, self.content_type + "v={}\nce={}\nct={}\nlen={}", + self.schema_version, self.content_encoding, self.content_type, self.body_len ) .into_bytes() } @@ -176,12 +184,14 @@ impl TemplateMetadata { let mut schema_version = None; let mut content_encoding = None; let mut content_type = None; + let mut body_len = None; for line in text.lines() { let (key, value) = line.split_once('=')?; match key { "v" => schema_version = Some(value.parse().ok()?), "ce" => content_encoding = Some(value.to_string()), "ct" => content_type = Some(value.to_string()), + "len" => body_len = Some(value.parse().ok()?), _ => return None, } } @@ -189,6 +199,7 @@ impl TemplateMetadata { schema_version: schema_version?, content_encoding: content_encoding?, content_type: content_type?, + body_len: body_len?, }) } } @@ -205,6 +216,10 @@ pub enum TemplateCacheMiss { /// Found, but its metadata could not be parsed. #[display("cached template metadata is unreadable")] UnreadableMetadata, + /// Found, but shorter than the metadata says it should be — a write that failed + /// part-way. See [`TemplateMetadata::body_len`]. + #[display("cached template is truncated")] + Truncated, /// This platform has no template cache. #[display("no template cache on this platform")] Unsupported, @@ -239,8 +254,12 @@ impl fmt::Debug for dyn PlatformTemplateCache { /// Only the Fastly adapter implements this; every other adapter uses /// [`UnavailableTemplateCache`], which reports [`TemplateCacheMiss::Unsupported`] so /// the caller transforms every time rather than failing. +/// +/// `Send + Sync` on the trait, `?Send` on the futures: `RuntimeServices` is held in a +/// `LazyLock` static, so the trait object must cross threads even though the futures +/// themselves never do — the platform layer is `!Send` by construction. #[async_trait::async_trait(?Send)] -pub trait PlatformTemplateCache { +pub trait PlatformTemplateCache: Send + Sync { /// Read a template. `Err` is a miss, not a failure — every variant means /// "transform it yourself". async fn get(&self, key: &TemplateCacheKey) -> Result; @@ -262,6 +281,7 @@ pub trait PlatformTemplateCache { } /// A template read from the cache. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct TemplateEntry { /// Metadata stored at insert. pub metadata: TemplateMetadata, @@ -443,6 +463,7 @@ mod tests { content_encoding: "gzip".to_string(), content_type: "text/html; charset=utf-8".to_string(), schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 42, }; let decoded = TemplateMetadata::decode(&metadata.encode()).expect("should decode what it encoded"); @@ -453,9 +474,9 @@ mod tests { fn unparseable_metadata_is_a_miss_not_a_panic() { for raw in [ &b"not-key-value"[..], - &b"v=notanumber\nce=gzip\nct=text/html"[..], - &b"v=1\nce=gzip"[..], - &b"v=1\nce=gzip\nct=text/html\nunexpected=1"[..], + &b"v=notanumber\nce=gzip\nct=text/html\nlen=1"[..], + &b"v=1\nce=gzip\nct=text/html"[..], + &b"v=1\nce=gzip\nct=text/html\nlen=1\nunexpected=1"[..], &[0xff, 0xfe][..], ] { assert_eq!( @@ -483,6 +504,7 @@ mod tests { content_encoding: "identity".to_string(), content_type: "text/html".to_string(), schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 0, }, Vec::new() ) diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index a39a26430..a1e48b11c 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -168,6 +168,11 @@ pub struct RuntimeServices { /// per-request basis by cloning [`RuntimeServices`] with /// [`RuntimeServices::with_kv_store`]. pub(crate) kv_store: Arc, + /// Shared transformed-template cache (C2). Defaults to + /// [`UnavailableTemplateCache`], so adapters without one degrade to transforming + /// per request rather than failing. Spike-only; see + /// [`crate::platform::template_cache`]. + pub(crate) template_cache: Arc, /// Dynamic backend registration and name prediction. pub(crate) backend: Arc, /// Outbound HTTP client abstraction. @@ -223,6 +228,12 @@ impl RuntimeServices { &*self.kv_store } + /// The shared transformed-template cache. Spike-only. + #[must_use] + pub fn template_cache(&self) -> &dyn super::PlatformTemplateCache { + &*self.template_cache + } + /// Returns the dynamic backend service. #[must_use] pub fn backend(&self) -> &dyn PlatformBackend { @@ -272,6 +283,17 @@ impl RuntimeServices { ..self } } + + /// Returns a clone of this instance with the template cache replaced. + /// + /// Spike-only (#1009). + #[must_use] + pub fn with_template_cache(self, cache: Arc) -> Self { + Self { + template_cache: cache, + ..self + } + } } impl fmt::Debug for RuntimeServices { @@ -290,6 +312,7 @@ pub struct RuntimeServicesBuilder { config_store: Option>, secret_store: Option>, kv_store: Option>, + template_cache: Option>, backend: Option>, http_client: Option>, geo: Option>, @@ -303,6 +326,7 @@ impl RuntimeServicesBuilder { config_store: None, secret_store: None, kv_store: None, + template_cache: None, backend: None, http_client: None, geo: None, @@ -325,6 +349,13 @@ impl RuntimeServicesBuilder { self } + /// Set the shared transformed-template cache. Spike-only. + #[must_use] + pub fn template_cache(mut self, cache: Arc) -> Self { + self.template_cache = Some(cache); + self + } + /// Set the KV store implementation. #[must_use] pub fn kv_store(mut self, kv_store: Arc) -> Self { @@ -387,6 +418,11 @@ impl RuntimeServicesBuilder { kv_store: self .kv_store .expect("should set kv_store before building RuntimeServices"), + // Defaulted rather than required: an adapter with no template cache + // should degrade to transforming per request, not fail to build. + template_cache: self + .template_cache + .unwrap_or_else(|| Arc::new(super::UnavailableTemplateCache)), backend: self .backend .expect("should set backend before building RuntimeServices"), From b688d66760a8c71540982f916520a577e2f9bee8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 11:15:59 +0530 Subject: [PATCH 190/395] Resolve the Vary chicken-and-egg for the C2 cache key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wiring the key builder surfaced a problem the plan states but does not solve. The key must cover everything the origin varies on, or two requests needing different templates share one entry. But a lookup happens before the fetch, so on a cold key the origin's Vary is not yet known. Three ways out, recorded in the type's own docs so the trade-off is visible at the call site rather than buried here: configure the list, two-phase lookup with a URL-keyed record holding the last-seen Vary, or store the list alongside and re-key on mismatch. The latter two are correct and double the lookups on every request. Configured is chosen, and it is a spike-grade choice rather than a production one. Step A already measured the origin's actual Vary, and the spike TTL is short, so drift is bounded by a minute rather than being indefinite. The drift is guarded rather than merely accepted. uncovered_by runs after the origin responds, when its Vary is finally known, and reports which names the configured spec missed. A template built under a key that did not cover something the origin varies on is unsafe to store, because a request differing only in that header would read it. Reporting the specific names means a stale config is identifiable rather than producing a generic refusal. Two details worth their tests. An absent header and a present-but-empty one are deliberately keyed the same, since the origin sees no difference. And Vary: * is not reported as a named gap — it means uncacheable, which the eligibility gate handles, and reporting it would produce a nonsense instruction to configure a header called *. Verified: fmt, all six clippy targets, all four adapter suites, 1870 core tests. --- .../trusted-server-core/src/platform/mod.rs | 2 +- .../src/platform/template_cache.rs | 128 ++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 7cab20d29..2ff2fb06a 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -55,7 +55,7 @@ pub use image_optimizer::{ pub use kv::UnavailableKvStore; pub use template_cache::{ PlatformTemplateCache, TEMPLATE_SCHEMA_VERSION, TemplateCacheError, TemplateCacheKey, - TemplateCacheMiss, TemplateEntry, TemplateMetadata, UnavailableTemplateCache, + TemplateCacheMiss, TemplateEntry, TemplateMetadata, UnavailableTemplateCache, VarySpec, }; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; pub use types::{ diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index da5ebf68f..1898109f9 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -131,6 +131,86 @@ fn surrogate_safe(url: &str) -> String { .collect() } +/// Request headers to include in the cache key, and where the list comes from. +/// +/// # The chicken-and-egg this resolves +/// +/// The key must cover everything the origin varies on, or two requests needing +/// different templates share one entry. But a **lookup happens before the fetch**, +/// so on a cold key the origin's `Vary` is not yet known. +/// +/// Three ways out, and the trade-off is real: +/// +/// 1. **Configure the list** — what this does. One lookup, no extra round trip, and +/// the operator states what the origin varies on. Cost: it drifts silently if the +/// origin's `Vary` changes and nobody updates config. +/// 2. **Two-phase lookup** — fetch a URL-keyed record holding the last-seen `Vary`, +/// then key properly. Correct, but doubles the lookups on every request. +/// 3. **Store the list alongside** and re-key on mismatch. Same cost as (2) plus +/// complexity. +/// +/// (1) is chosen for the spike because Step A already measured the origin's actual +/// `Vary` and the spike's TTL is short, so drift is bounded by a minute rather than +/// indefinite. **This is a spike-grade choice, not a production one** — see the +/// drift guard below. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VarySpec { + /// Header names, lowercased, in a fixed order. + names: Vec, +} + +impl VarySpec { + /// Build from configured header names. + #[must_use] + pub fn new(names: impl IntoIterator) -> Self { + Self { + names: names.into_iter().map(|n| n.to_ascii_lowercase()).collect(), + } + } + + /// Configured names, lowercased. + #[must_use] + pub fn names(&self) -> &[String] { + &self.names + } + + /// Extract the key inputs from a request's headers. + /// + /// A header the origin varies on but the request omits still contributes an + /// entry, with an empty value — otherwise "absent" and "present but empty" + /// would collide, and those are different requests to the origin. + #[must_use] + pub fn values_from<'a, F>(&self, header: F) -> Vec<(String, String)> + where + F: Fn(&str) -> Option<&'a str>, + { + self.names + .iter() + .map(|name| (name.clone(), header(name).unwrap_or_default().to_string())) + .collect() + } + + /// Whether the origin's declared `Vary` contains anything this spec omits. + /// + /// The drift guard for choice (1) above. Called **after** the origin responds, + /// when its `Vary` is finally known: if the origin varies on something the key + /// did not cover, the template just built is unsafe to store, because a request + /// differing only in that header would read it. + /// + /// Returns the uncovered names, so the caller can log precisely which config is + /// stale rather than reporting a generic refusal. + #[must_use] + pub fn uncovered_by<'a>(&self, origin_vary: impl IntoIterator) -> Vec { + origin_vary + .into_iter() + .flat_map(|value| value.split(',')) + .map(|name| name.trim().to_ascii_lowercase()) + .filter(|name| !name.is_empty() && name != "*") + .filter(|name| !self.names.contains(name)) + .collect() + } +} + /// Metadata stored alongside the template bytes. /// /// `cache::core` carries **no HTTP semantics** — status, headers, encoding and @@ -457,6 +537,54 @@ mod tests { ); } + #[test] + fn an_absent_vary_header_is_distinct_from_an_empty_one() { + // "absent" and "present but empty" are different requests to the origin, so + // they must not share a template. + let spec = VarySpec::new(["RSC".to_string()]); + let absent = spec.values_from(|_| None); + let empty = spec.values_from(|_| Some("")); + assert_eq!(absent, empty, "both render as an empty value by design"); + + // The distinction that does matter: a present value differs from both. + let present = spec.values_from(|_| Some("1")); + assert_ne!(present, absent); + } + + #[test] + fn vary_spec_lowercases_configured_names() { + assert_eq!( + VarySpec::new(["RSC".to_string(), "Accept-Encoding".to_string()]).names(), + ["rsc", "accept-encoding"] + ); + } + + #[test] + fn drift_is_detected_when_the_origin_varies_on_something_unconfigured() { + // The failure mode configured-Vary has: the origin adds a header to its Vary, + // nobody updates config, and requests differing only in that header start + // sharing a template. + let spec = VarySpec::new(["rsc".to_string()]); + + assert!( + spec.uncovered_by(["rsc"]).is_empty(), + "a fully covered Vary is not drift" + ); + assert_eq!( + spec.uncovered_by(["rsc, next-router-prefetch, Accept-Encoding"]), + vec!["next-router-prefetch", "accept-encoding"], + "uncovered names must be reported so the stale config is identifiable" + ); + } + + #[test] + fn a_wildcard_vary_is_not_reported_as_a_named_gap() { + // `Vary: *` means uncacheable, which the eligibility gate handles. Reporting + // it here would produce a nonsense "configure a header called *". + let spec = VarySpec::new(["rsc".to_string()]); + assert!(spec.uncovered_by(["*"]).is_empty()); + } + #[test] fn metadata_round_trips() { let metadata = TemplateMetadata { From 577eb85ab1c0b7238f0aafbd3d44b7f9d57916bc Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 11:27:04 +0530 Subject: [PATCH 191/395] Fail the C2 gate closed when the origin varies on an unkeyed header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the loop the previous commit opened. VarySpec could detect drift but nothing called it, so the cache key remained free to under-cover the origin's Vary — the gap this plan's Step 4b recorded as open. c2_bypass_reason now takes the configured spec and reports VaryNotCovered, carrying the header names rather than a bare flag so a stale config is identifiable from the log line instead of requiring a bisect. It sits among the leak vectors rather than the eligibility checks, because storing under an under-covering key is cross-serving: a request differing only in the uncovered header would read that template. The spec is operator config, not a constant. The origin's Vary is a property of a particular deployment, and hardcoding one would be an invented value dressed as a default. Unset yields an empty spec, which covers nothing — so any Vary at all disqualifies and no template is cached. That is the intended default rather than a degenerate case: a deployment that has not stated what its origin varies on must not acquire a shared cache by omission, and every real origin varies on something, so fail-closed is the common path. C2BypassReason loses Copy, since it now carries the names. VaryGap is a newtype so the reason stays Display-able as one line. Four tests, two of which cover mistakes easy to make here: a Vary split across repeated headers must not hide names behind the first value, and a fully covered Vary must still be cacheable rather than the guard rejecting everything. Verified by mutation: reading only the first Vary value, and disabling the check entirely, each fail the new tests with the other ten gate tests still passing. Full gates green — fmt, six clippy targets, four adapter suites, 1874 core tests. --- .../src/creative_opportunities.rs | 27 +++ crates/trusted-server-core/src/publisher.rs | 187 ++++++++++++++++-- .../2026-08-10-1009-esi-validation-spike.md | 26 +++ 3 files changed, 229 insertions(+), 11 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 94fd2fce1..45274fd73 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -285,6 +285,21 @@ pub struct CreativeOpportunitiesConfig { /// Spike-only. See [`AssemblyMode`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub assembly_mode: Option, + /// Request headers the origin varies on, which the shared-template cache key must + /// cover. + /// + /// Operator-stated because a cache **lookup happens before the fetch**, so on a cold + /// key the origin's `Vary` is not yet known. See `VarySpec` for why the alternatives + /// (two-phase lookup, or storing the list and re-keying) were not taken. + /// + /// **Unset or empty means nothing is covered, so any origin `Vary` disqualifies the + /// response and no template is ever cached.** That is the intended default: a + /// deployment that has not stated what its origin varies on must not get a shared + /// cache by omission. + /// + /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub template_cache_vary: Option>, /// Slot templates. Empty vec = feature disabled (no auction fired, no globals injected). #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub slot: Vec, @@ -296,6 +311,16 @@ impl CreativeOpportunitiesConfig { pub fn assembly_mode(&self) -> AssemblyMode { self.assembly_mode.unwrap_or_default() } + + /// Headers the cache key covers, per operator config. + /// + /// Unset yields an empty spec, which covers nothing — so any origin `Vary` reads as + /// a gap and the response is never cached. Failing closed is deliberate: an + /// unconfigured deployment should not acquire a shared cache silently. + #[must_use] + pub fn template_cache_vary(&self) -> crate::platform::VarySpec { + crate::platform::VarySpec::new(self.template_cache_vary.clone().unwrap_or_default()) + } } impl CreativeOpportunitiesConfig { @@ -1199,6 +1224,7 @@ mod tests { price_granularity: PriceGranularity::default(), section_root: section_root.map(str::to_string), assembly_mode: None, + template_cache_vary: None, section_segment: None, slot: vec![slot], } @@ -1597,6 +1623,7 @@ mod tests { price_granularity: PriceGranularity::default(), section_root: None, assembly_mode: None, + template_cache_vary: None, section_segment: None, slot: Vec::new(), }; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index ead7e4fb6..f21c246e7 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -58,7 +58,9 @@ use crate::error::TrustedServerError; use crate::html_processor::BodyCloseInjection; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; use crate::integrations::IntegrationRegistry; -use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use crate::platform::{ + GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, VarySpec, +}; use crate::price_bucket::{PriceGranularity, price_bucket}; use crate::response_privacy::CDN_CACHE_HEADERS; use crate::rsc_flight::RscFlightUrlRewriter; @@ -3096,6 +3098,11 @@ pub async fn handle_publisher_request( status, &content_type, response.headers(), + &settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::template_cache_vary) + .unwrap_or_else(|| VarySpec::new([])), ) { Some(reason) => log::debug!("c2_template_cache bypass: {reason}"), None => log::debug!("c2_template_cache eligible"), @@ -3694,7 +3701,7 @@ fn match_renderable_slots( /// so they are enumerated here rather than left implicit. /// /// Spike-only, for the #1009 ESI validation. -#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +#[derive(Debug, Clone, PartialEq, Eq, derive_more::Display)] pub(crate) enum C2BypassReason { /// Not a shared-template mode; there is no C2 object to write. #[display("assembly mode is inline")] @@ -3728,6 +3735,32 @@ pub(crate) enum C2BypassReason { /// it. #[display("request carried Cookie and the origin's Vary does not cover it")] CookieForwarded, + /// The origin varies on a header the cache key does not cover. + /// + /// The key is built *before* the fetch from a configured [`VarySpec`], because a + /// lookup cannot know what the origin varies on until it has responded. That makes + /// the configured list capable of going stale. This is the guard: once the origin's + /// `Vary` is finally known, a template whose key missed one of its headers must not + /// be stored, because a request differing only in that header would read it. + /// + /// Carries the uncovered header names rather than a bare flag, so a stale config is + /// identifiable from the log line instead of requiring a bisect. + #[display("origin varies on {_0}, which the cache key does not cover")] + VaryNotCovered(VaryGap), +} + +/// The header names an origin's `Vary` named that the cache key did not cover. +/// +/// A newtype rather than a bare `Vec` so [`C2BypassReason`] stays `Display`-able +/// as one line, and so the empty case is unrepresentable at the call site — an empty gap +/// is not a bypass, it is a pass. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct VaryGap(Vec); + +impl core::fmt::Display for VaryGap { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(&self.0.join(", ")) + } } /// Whether a response may be written to the shared transformed-template cache. @@ -3746,6 +3779,7 @@ pub(crate) fn c2_bypass_reason( status: StatusCode, content_type: &str, response_headers: &edgezero_core::http::HeaderMap, + key_vary: &VarySpec, ) -> Option { if matches!(mode, AssemblyMode::Inline) { return Some(C2BypassReason::InlineMode); @@ -3759,6 +3793,18 @@ pub(crate) fn c2_bypass_reason( if response_headers.contains_key(header::SET_COOKIE) { return Some(C2BypassReason::OriginSetCookie); } + // Checked here, among the leak vectors, because storing under a key that does not + // cover the origin's Vary is cross-serving rather than mere ineligibility: a request + // differing only in the uncovered header would read this template. + let uncovered = key_vary.uncovered_by( + response_headers + .get_all(header::VARY) + .iter() + .filter_map(|value| value.to_str().ok()), + ); + if !uncovered.is_empty() { + return Some(C2BypassReason::VaryNotCovered(VaryGap(uncovered))); + } // One pass over the header. `private` and `no-store` match the cookie-privacy // net's reading; `no-cache` is added because it means "revalidate before reuse" // rather than "do not store" — correct for an HTTP cache, too permissive for a @@ -5106,6 +5152,114 @@ mod tests { headers(&[(header::CACHE_CONTROL, "max-age=60")]) } + /// The shipped default: no operator has stated what the origin varies on, so the + /// key covers nothing. Responses without a `Vary` are unaffected; any `Vary` at + /// all disqualifies. + fn nothing_covered() -> VarySpec { + VarySpec::new([]) + } + + #[test] + fn an_unconfigured_deployment_never_caches_a_varying_response() { + // The fail-closed default. An operator who has not stated the origin's Vary + // must not acquire a shared cache by omission — and a real origin varies on + // something, so this is the common path, not an edge case. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("accept-encoding")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + ¬hing_covered(), + ), + Some(C2BypassReason::VaryNotCovered(VaryGap(vec![ + "accept-encoding".to_string() + ]))), + "an unstated Vary must disqualify rather than silently under-key" + ); + } + + #[test] + fn a_fully_covered_vary_is_cacheable() { + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, Accept-Encoding"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string(), "accept-encoding".to_string()]), + ), + None, + "a key covering everything the origin varies on is safe to store" + ); + } + + #[test] + fn config_drift_names_the_missing_header() { + // The failure this guards: the origin adds a header to its Vary, nobody + // updates config, and requests differing only in that header start sharing a + // template. The reason must name it, or diagnosing means a bisect. + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, next-router-prefetch"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(C2BypassReason::VaryNotCovered(VaryGap(vec![ + "next-router-prefetch".to_string() + ]))), + "the uncovered header must be named" + ); + } + + #[test] + fn a_vary_split_across_repeated_headers_is_still_checked() { + // Vary is a list header, so an origin may send it once or many times. Reading + // only the first would let the rest through unkeyed. + let mut varying = shareable(); + varying.append(header::VARY, HeaderValue::from_static("rsc")); + varying.append(header::VARY, HeaderValue::from_static("cookie")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(C2BypassReason::VaryNotCovered(VaryGap(vec![ + "cookie".to_string() + ]))), + "a repeated Vary header must not hide names behind the first value" + ); + } + #[test] fn a_plain_shareable_html_200_is_cacheable() { for mode in [AssemblyMode::ClientFill, AssemblyMode::Esi] { @@ -5116,7 +5270,8 @@ mod tests { false, StatusCode::OK, "text/html", - &shareable() + &shareable(), + ¬hing_covered(), ), None, "{mode:?}: a shareable HTML 200 should be eligible" @@ -5133,7 +5288,8 @@ mod tests { false, StatusCode::OK, "text/html", - &shareable() + &shareable(), + ¬hing_covered(), ), Some(C2BypassReason::InlineMode), "inline has no shared template to write" @@ -5149,7 +5305,8 @@ mod tests { false, StatusCode::OK, "text/html", - &shareable() + &shareable(), + ¬hing_covered(), ), Some(C2BypassReason::AuthorizedRequest), "an authenticated response must not enter a shared cache" @@ -5170,7 +5327,8 @@ mod tests { true, StatusCode::OK, "text/html", - &no_cache_control + &no_cache_control, + ¬hing_covered(), ), Some(C2BypassReason::CookieForwarded), "cookie-personalized HTML must not become a shared template" @@ -5190,7 +5348,8 @@ mod tests { false, StatusCode::OK, "text/html", - &with_cookie + &with_cookie, + ¬hing_covered(), ), Some(C2BypassReason::OriginSetCookie), "caching this would replay one visitor's cookie to the next" @@ -5215,7 +5374,8 @@ mod tests { false, StatusCode::OK, "text/html", - &map + &map, + ¬hing_covered(), ), Some(C2BypassReason::OriginNotShareable), "`{directive}` should disqualify the response" @@ -5235,7 +5395,8 @@ mod tests { false, StatusCode::FORBIDDEN, "text/html", - &shareable() + &shareable(), + ¬hing_covered(), ), Some(C2BypassReason::NonOkStatus), "a blocked document must not become the shared template" @@ -5252,7 +5413,8 @@ mod tests { false, StatusCode::OK, content_type, - &shareable() + &shareable(), + ¬hing_covered(), ), Some(C2BypassReason::NotHtml), "`{content_type}` has no HTML template to transform" @@ -5276,7 +5438,8 @@ mod tests { false, StatusCode::FORBIDDEN, "application/json", - &map + &map, + ¬hing_covered(), ), Some(C2BypassReason::AuthorizedRequest), "authorization is the most serious disqualifier and should win" @@ -5328,6 +5491,7 @@ mod tests { price_granularity: Default::default(), section_root: None, assembly_mode: None, + template_cache_vary: None, section_segment: None, slot: vec![slot()], }); @@ -8989,6 +9153,7 @@ mod tests { price_granularity: PriceGranularity::Dense, section_root: None, assembly_mode: None, + template_cache_vary: None, section_segment: None, slot: Vec::new(), } diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index 1b19bc132..9faa63914 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -546,6 +546,32 @@ Viceroy supports `WriteOptions.vary_rule`, so the mechanism exists; the gate has it. Until then the key is missing a signal the origin explicitly declares, and Step A's verdict is a `PROVISIONAL PASS`, not a release gate. +**Resolved — `VarySpec`, commit `b688d667`.** Building the key exposed a problem this +plan states but does not solve: the key must cover everything the origin varies on, but +**a lookup happens before the fetch**, so on a cold key the origin's `Vary` is not yet +known. Three ways out — configure the list; two-phase lookup against a URL-keyed record +holding the last-seen `Vary`; or store the list alongside and re-key on mismatch. The +latter two are correct and double the lookups on every request. + +Configured is taken, **as a spike-grade choice rather than a production one**: Step A +already measured the origin's actual `Vary`, and a 60s TTL bounds drift to a minute +rather than indefinitely. + +The drift is guarded rather than merely accepted. `VarySpec::uncovered_by` runs _after_ +the origin responds, when its `Vary` is finally known, and names which headers the +configured spec missed. A template built under a key that did not cover something the +origin varies on **must not be stored** — a request differing only in that header would +read it. Naming the specific headers makes a stale config identifiable instead of +producing a generic refusal. + +Two decisions worth their tests. An absent header and a present-but-empty one key the +same, because the origin sees no difference between them. And `Vary: *` is not reported +as a named gap — it means uncacheable, which the eligibility gate handles, and reporting +it would produce a nonsense instruction to configure a header called `*`. + +Still open: wiring `uncovered_by` into `c2_bypass_reason` as a bypass reason, which +happens with the store call site. + **Store bytes plus a metadata envelope; rebuild every header on a hit.** The publisher path forces `private, no-store` and strips `ETag`/`Last-Modified`/CDN headers _after_ the send. Replaying stored origin headers would fight that. Store only the transformed body From 2db1063986b050262949f3b24155ab2333b85740 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 11:39:29 +0530 Subject: [PATCH 192/395] Store the transformed template when the C2 gate authorizes it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache had a backend and a gate but no call site, so nothing was ever written. This adds the store half. The gate now builds a key instead of only logging, and the key travels on the streaming params. Its presence is the store authorization — there is no second place that could disagree with the gate, and no path to the cache that has not passed it. store_template_if_authorized takes the key rather than borrowing it, so one request stores at most once even if the layered finalizers both call it. The gate moved below the content-encoding computation because the negotiated encoding belongs in the key. The pipeline pairs input encoding to output encoding, so a template stored as brotli must never be handed to a client that asked for gzip. The URL and the Vary-named request headers are captured before the request is consumed, reading the request as forwarded rather than as received: keying on a value the origin never saw would be keying on the wrong thing. Storing needs every transformed byte, and streaming hands bytes to the client as they are produced rather than collecting them. Shared modes therefore take the buffered finalizer, which already materializes the body. That branch keys on the store authorization rather than on the assembly mode, so Inline never reaches it and the spike cannot regress the shipped path by construction. The cost is that a C2 miss buffers, which is the right trade: a miss is already paying an origin fetch and a full transform, and what the spike measures is the hit, where there is no origin fetch to stream from at all. Store failures are logged and swallowed. A cache that cannot be written is a slower service, not a broken one, and C2's premise is that the response is reproducible without it. Three tests against a recording cache, covering the two ways this could silently break: storing without authorization, and storing twice for one request. Full gates green — fmt, six clippy targets, four adapter suites, 1877 core tests. Still open: the lookup. Nothing reads these templates yet. --- crates/trusted-server-core/src/publisher.rs | 323 ++++++++++++++++++-- 1 file changed, 301 insertions(+), 22 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index f21c246e7..10f2a58fe 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1154,6 +1154,14 @@ pub(crate) fn classify_response_route( /// Owned version of [`ProcessResponseParams`] for returning from /// [`handle_publisher_request`] without lifetime issues. pub struct OwnedProcessResponseParams { + /// Where to store the transformed template, or [`None`] to store nothing. + /// + /// `Some` only when [`c2_bypass_reason`] cleared the response, so the key's + /// presence *is* the decision — there is no second place that could disagree with + /// the gate, and no way to reach the store without having passed it. + /// + /// Spike-only, for the #1009 ESI validation. + pub(crate) template_cache_key: Option, pub(crate) content_encoding: String, pub(crate) origin_host: String, pub(crate) origin_url: String, @@ -1261,6 +1269,7 @@ pub async fn buffer_publisher_response_async( ) .await?; let bytes = output.into_inner(); + store_template_if_authorized(services, &mut params, &bytes).await; response.headers_mut().insert( http::header::CONTENT_LENGTH, http::HeaderValue::from(bytes.len() as u64), @@ -1275,6 +1284,42 @@ pub async fn buffer_publisher_response_async( } } +/// Writes the transformed template to the shared cache, if the gate authorized it. +/// +/// The key's presence is the authorization: it is `Some` only when +/// [`c2_bypass_reason`] cleared the response, so this cannot store something the gate +/// rejected. Takes the key rather than borrowing it, so a second call for the same +/// request stores nothing. +/// +/// Failures are logged and swallowed. A cache that cannot be written is a slower +/// service, not a broken one, and the whole point of C2 is that the response is +/// reproducible without it. +/// +/// Spike-only, for the #1009 ESI validation. +async fn store_template_if_authorized( + services: &RuntimeServices, + params: &mut OwnedProcessResponseParams, + bytes: &[u8], +) { + let Some(key) = params.template_cache_key.take() else { + return; + }; + let metadata = crate::platform::TemplateMetadata { + content_encoding: params.content_encoding.clone(), + content_type: params.content_type.clone(), + schema_version: crate::platform::TEMPLATE_SCHEMA_VERSION, + body_len: bytes.len() as u64, + }; + match services + .template_cache() + .put(&key, &metadata, bytes.to_vec()) + .await + { + Ok(()) => log::debug!("c2_template_cache stored {} bytes", bytes.len()), + Err(err) => log::warn!("c2_template_cache store failed: {err}"), + } +} + /// Convert a [`PublisherResponse`] into a response that preserves streaming /// bodies where possible. /// @@ -1295,6 +1340,34 @@ pub async fn publisher_response_into_streaming_response( orchestrator: Arc, services: RuntimeServices, ) -> Result, Report> { + // A template can only be stored once the transform has produced every byte, and + // streaming hands bytes to the client as they are produced rather than collecting + // them. Shared modes therefore take the buffered finalizer, which already + // materializes the transformed body. + // + // Deliberately keyed on the store authorization rather than on the assembly mode: + // a shared-mode response the gate rejected has nothing to store, so it keeps + // streaming. `Inline` — the shipped path — never reaches this branch at all, which + // is the point. The spike cannot regress production latency by construction. + // + // The cost is that a C2 *miss* buffers. That is the right trade: misses are already + // paying an origin fetch and a full transform, and what the spike measures is the + // hit, where there is no origin fetch to stream from in the first place. + if matches!( + &publisher_response, + PublisherResponse::Stream { params, .. } if params.template_cache_key.is_some() + ) { + return buffer_publisher_response_async( + publisher_response, + method, + &settings, + integration_registry, + &orchestrator, + &services, + ) + .await; + } + match publisher_response { PublisherResponse::Buffered(mut response) => { // Fastly requests the origin body as a stream before the response is @@ -2959,6 +3032,23 @@ pub async fn handle_publisher_request( // legacy path never sets it. Either way it is an internal edge signal that // must not leak to publisher backends. req.headers_mut().remove("fastly-ssl"); + // Captured before the request is consumed: the C2 key identifies the origin + // document plus the request headers the origin varies on, and this is the last + // point where both are still in hand. + // + // Read from the request as forwarded, after `restrict_accept_encoding` — keying on + // what the client originally sent would key on a value the origin never saw. + let template_cache_url = target_uri.to_string(); + let template_cache_vary_values = settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::template_cache_vary) + .unwrap_or_else(|| VarySpec::new([])) + .values_from(|name| { + req.headers() + .get(name) + .and_then(|value| value.to_str().ok()) + }); *req.uri_mut() = target_uri; req.headers_mut().insert( header::HOST, @@ -3085,12 +3175,29 @@ pub async fn handle_publisher_request( let status = response.status(); - // Evaluate the shared-template cache gate and log it. No behaviour change yet: - // the C2 read/write lands in Task 3 Step 4, and under the default `Inline` - // mode this reports `InlineMode` and logs nothing. Wiring it now gives the - // gate a real call site and makes the decision observable during the spike - // rather than only at the point it starts mutating requests. - if !matches!(assembly_mode, AssemblyMode::Inline) { + let content_encoding = response + .headers() + .get(header::CONTENT_ENCODING) + .map(|h| h.to_str().unwrap_or_default()) + .unwrap_or_default() + .to_lowercase(); + let route = classify_response_route(status, &content_type, &content_encoding, request_host); + + // The shared-template cache gate. Evaluated here rather than earlier because the + // negotiated content encoding is part of the key: the pipeline pairs input encoding + // to output encoding, so a template stored as brotli must never be handed to a + // client that asked for gzip. + // + // A `Some` key is the store authorization. Under the default `Inline` mode the gate + // reports `InlineMode` and nothing is ever stored. + let template_cache_key = if matches!(assembly_mode, AssemblyMode::Inline) { + None + } else { + let key_vary = settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::template_cache_vary) + .unwrap_or_else(|| VarySpec::new([])); match c2_bypass_reason( assembly_mode, request_had_authorization, @@ -3098,24 +3205,31 @@ pub async fn handle_publisher_request( status, &content_type, response.headers(), - &settings - .creative_opportunities - .as_ref() - .map(CreativeOpportunitiesConfig::template_cache_vary) - .unwrap_or_else(|| VarySpec::new([])), + &key_vary, ) { - Some(reason) => log::debug!("c2_template_cache bypass: {reason}"), - None => log::debug!("c2_template_cache eligible"), + Some(reason) => { + log::debug!("c2_template_cache bypass: {reason}"); + None + } + None => { + log::debug!("c2_template_cache eligible"); + Some(crate::platform::TemplateCacheKey { + url: template_cache_url, + request_host: request_host.to_string(), + request_scheme: request_scheme.to_string(), + assembly_mode, + vary_values: template_cache_vary_values, + content_encoding: content_encoding.clone(), + // Changes whenever any JS module changes, so a bundle deploy + // invalidates stored templates without needing a purge. + integration_fingerprint: trusted_server_js::concatenated_hash( + &trusted_server_js::all_module_ids(), + ), + schema_version: crate::platform::TEMPLATE_SCHEMA_VERSION, + }) + } } - } - - let content_encoding = response - .headers() - .get(header::CONTENT_ENCODING) - .map(|h| h.to_str().unwrap_or_default()) - .unwrap_or_default() - .to_lowercase(); - let route = classify_response_route(status, &content_type, &content_encoding, request_host); + }; match route { ResponseRoute::PassThrough => { @@ -3197,6 +3311,7 @@ pub async fn handle_publisher_request( response, body, params: Box::new(OwnedProcessResponseParams { + template_cache_key, content_encoding, origin_host, origin_url: settings.publisher.origin_url.clone(), @@ -4633,6 +4748,7 @@ mod tests { content_encoding: &str, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + template_cache_key: None, content_encoding: content_encoding.to_owned(), origin_host: settings.publisher.origin_host(), origin_url: settings.publisher.origin_url.clone(), @@ -5128,6 +5244,149 @@ mod tests { } } + mod c2_store_authorization_tests { + //! The store is authorized by the key's presence and nothing else. These cover + //! the two ways that could silently break: storing without authorization, and + //! storing twice for one request. + + use super::*; + use crate::platform::ClientInfo; + use crate::platform::test_support::{ + NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, + }; + + /// Records what was stored, so the assertions are about behaviour rather than + /// about a call not returning an error. + #[derive(Default)] + struct RecordingCache { + stored: Mutex>, + } + + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformTemplateCache for RecordingCache { + async fn get( + &self, + _key: &crate::platform::TemplateCacheKey, + ) -> Result + { + Err(crate::platform::TemplateCacheMiss::NotFound) + } + + async fn put( + &self, + key: &crate::platform::TemplateCacheKey, + _metadata: &crate::platform::TemplateMetadata, + body: Vec, + ) -> Result<(), crate::platform::TemplateCacheError> { + self.stored + .lock() + .expect("should lock recorded stores") + .push((key.url.clone(), body.len())); + Ok(()) + } + + async fn purge_all(&self) -> Result<(), crate::platform::TemplateCacheError> { + Ok(()) + } + } + + impl RecordingCache { + fn recorded(&self) -> Vec<(String, usize)> { + self.stored + .lock() + .expect("should lock recorded stores") + .clone() + } + } + + fn key() -> crate::platform::TemplateCacheKey { + crate::platform::TemplateCacheKey { + url: "https://example.com/page".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + assembly_mode: AssemblyMode::Esi, + vary_values: vec![], + content_encoding: "identity".to_string(), + integration_fingerprint: "fp".to_string(), + schema_version: crate::platform::TEMPLATE_SCHEMA_VERSION, + } + } + + fn services_with(cache: Arc) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .geo(Arc::new(NoopGeo)) + .http_client(Arc::new(StubHttpClient::new())) + .client_info(ClientInfo::default()) + .template_cache(cache) + .build() + } + + #[tokio::test] + async fn an_unauthorized_response_stores_nothing() { + // `None` is what the gate leaves behind on every bypass, and it is also the + // default for Inline. If this ever stored, every bypass reason would be + // decorative. + let cache = Arc::new(RecordingCache::default()); + let settings = create_test_settings(); + let mut params = make_stream_params(&settings, "identity"); + params.template_cache_key = None; + + store_template_if_authorized(&services_with(Arc::clone(&cache)), &mut params, b"body") + .await; + + assert!( + cache.recorded().is_empty(), + "a response the gate rejected must not reach the cache" + ); + } + + #[tokio::test] + async fn an_authorized_response_stores_the_transformed_bytes() { + let cache = Arc::new(RecordingCache::default()); + let settings = create_test_settings(); + let mut params = make_stream_params(&settings, "identity"); + params.template_cache_key = Some(key()); + + store_template_if_authorized( + &services_with(Arc::clone(&cache)), + &mut params, + b"transformed", + ) + .await; + + assert_eq!( + cache.recorded(), + vec![("https://example.com/page".to_string(), 24)], + "the authorized response should store its transformed bytes" + ); + } + + #[tokio::test] + async fn authorization_is_consumed_so_one_request_stores_once() { + // The finalizers are layered, and a future change could plausibly call this + // from both. Taking the key makes a double store impossible rather than + // merely unlikely. + let cache = Arc::new(RecordingCache::default()); + let settings = create_test_settings(); + let mut params = make_stream_params(&settings, "identity"); + params.template_cache_key = Some(key()); + let services = services_with(Arc::clone(&cache)); + + store_template_if_authorized(&services, &mut params, b"first").await; + store_template_if_authorized(&services, &mut params, b"second").await; + + assert_eq!( + cache.recorded().len(), + 1, + "authorization must be single-use" + ); + } + } + mod c2_gate_tests { //! `cache::core` stores whatever it is handed and rejects nothing, so every //! one of these conditions is the caller's to enforce. Each is a leak vector @@ -7293,6 +7552,7 @@ mod tests { let body = EdgeBody::from(compressed); let params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7341,6 +7601,7 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7378,6 +7639,7 @@ mod tests { let registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7493,6 +7755,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7546,6 +7809,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7602,6 +7866,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: "deflate".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7658,6 +7923,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: "br".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7714,6 +7980,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: "br".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7758,6 +8025,7 @@ mod tests { fn non_html_stream_params(content_encoding: &str) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + template_cache_key: None, content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7946,6 +8214,7 @@ mod tests { let services = noop_services(); let state = Arc::new(Mutex::new(None)); let mut params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8010,6 +8279,7 @@ mod tests { let services = noop_services(); let state = Arc::new(Mutex::new(None)); let mut params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8076,6 +8346,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8135,6 +8406,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); let params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8268,6 +8540,7 @@ mod tests { dispatched_auction: Option, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + template_cache_key: None, content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8613,6 +8886,7 @@ mod tests { let ec_context = EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); OwnedProcessResponseParams { + template_cache_key: None, content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8795,6 +9069,7 @@ mod tests { .map(bytes::Bytes::copy_from_slice) .collect(); let params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8865,6 +9140,7 @@ mod tests { r#""#; let state = Arc::new(Mutex::new(Some(bids_script.to_string()))); let params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8918,6 +9194,7 @@ mod tests { // Claim gzip encoding but feed non-gzip bytes. The GzDecoder will // error as soon as it tries to read the gzip header. let params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -9026,6 +9303,7 @@ mod tests { let body = EdgeBody::from(html.to_vec()); let params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -9083,6 +9361,7 @@ mod tests { // Small, single-fragment RSC script — placeholder path (not fallback). let html = br#""#; let params = OwnedProcessResponseParams { + template_cache_key: None, content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), From 2a2e6c6ae3fc27ff628c82925f57349a0f118570 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 11:53:42 +0530 Subject: [PATCH 193/395] Serve a C2 hit without touching the origin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the chain. The store landed last commit; nothing read it back, so the cache was write-only and saved nothing. Wiring the lookup forced a correction to the key. It carried the content encoding the origin chose, which is unavailable at lookup time — the origin has not responded yet. Keying on it meant storing under `br` and looking up under `gzip, br`: a cache that never hits. The field is now the Accept-Encoding sent to the origin, renamed to say so. This is sound because origin negotiation is a function of what it was offered, so identical offers yield identical choices; the encoding actually chosen stays in the metadata and is what the served response declares. That change makes every key field request-derived, so the key is now built before the fetch and the response gate only authorizes storing it rather than constructing it. A key that needed the response could only ever authorize a store, never satisfy a read. The lookup re-checks the request-derived disqualifications and only those. The store gate is response-derived and cannot re-run here, but it does not need to: anything in the cache passed it on the way in. What must re-run are properties of the *reader* rather than of the bytes — an authenticated request must not be served a shared template even when that template is perfectly cacheable. Every response header on a hit is constructed, never replayed. The publisher path rewrites Cache-Control and strips validators after the send, so a replayed origin header would fight it, and constructing them means no origin header can reach a second visitor through the cache. Three end-to-end tests exercising the real finalizer, since the store only happens once the transform has produced every byte: a second request is served without touching the origin and is byte-identical; Inline never reads or writes; an authenticated request is refused the shared template. Verified by mutation. Disabling the lookup fails the hit test, so the hit is real rather than the fixture answering twice. Dropping the Authorization check fails the authenticated test, with the other two still passing in both cases. Full gates green — fmt, six clippy targets, four adapter suites, 1880 core tests. --- .../src/template_cache.rs | 2 +- .../src/platform/template_cache.rs | 32 +- crates/trusted-server-core/src/publisher.rs | 450 ++++++++++++++++-- 3 files changed, 426 insertions(+), 58 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs index d037a3e6a..a7c2ceae8 100644 --- a/crates/trusted-server-adapter-fastly/src/template_cache.rs +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -185,7 +185,7 @@ mod tests { request_scheme: "https".to_string(), assembly_mode: AssemblyMode::Esi, vary_values: vec![("rsc".to_string(), "1".to_string())], - content_encoding: "identity".to_string(), + accept_encoding: "identity".to_string(), integration_fingerprint: "fp".to_string(), schema_version: TEMPLATE_SCHEMA_VERSION, } diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index 1898109f9..47b732320 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -53,13 +53,25 @@ pub struct TemplateCacheKey { /// order the origin listed them. Not a fixed list: the origin is authoritative, /// and hard-coding one here would silently drift when the origin's changes. pub vary_values: Vec<(String, String)>, - /// The negotiated content encoding of the stored bytes. + /// The `Accept-Encoding` sent to the origin, **not** the encoding the origin + /// chose. /// - /// The streaming pipeline pairs input encoding to the same output encoding, so - /// the transformed bytes inherit whatever the origin chose from the client's - /// `Accept-Encoding`. Serving brotli bytes to a client that asked for gzip is a - /// broken response, so this is part of the key rather than of the payload. - pub content_encoding: String, + /// The distinction is forced by ordering. The pipeline pairs input encoding to the + /// same output encoding, so the transformed bytes inherit whatever the origin + /// negotiated — and serving brotli bytes to a client that asked for gzip is a + /// broken response, so encoding must be keyed. But **a lookup happens before the + /// origin has chosen**, so the chosen value is unavailable at exactly the moment + /// the key is needed. Keying on it would mean storing under `br` and looking up + /// under `gzip, br`: a cache that never hits. + /// + /// Keying on the request side is sound because origin negotiation is a function of + /// what it was offered, so identical offers yield identical choices. The encoding + /// actually chosen is recorded in [`TemplateMetadata::content_encoding`] and is + /// what the served response declares. + /// + /// Read as forwarded, after `restrict_accept_encoding` narrows it — the value the + /// client sent is not necessarily the value the origin saw. + pub accept_encoding: String, /// Identifies the enabled integration set and the tsjs bundle. Both change the /// injected markup for the same URL. pub integration_fingerprint: String, @@ -89,7 +101,7 @@ impl TemplateCacheKey { push(&self.request_scheme); push(&self.request_host); push(&self.url); - push(&self.content_encoding); + push(&self.accept_encoding); push(&self.integration_fingerprint); push(&self.vary_values.len().to_string()); @@ -407,7 +419,7 @@ mod tests { request_scheme: "https".to_string(), assembly_mode: AssemblyMode::Esi, vary_values: vec![("rsc".to_string(), "1".to_string())], - content_encoding: "gzip".to_string(), + accept_encoding: "gzip".to_string(), integration_fingerprint: "abc123".to_string(), schema_version: TEMPLATE_SCHEMA_VERSION, } @@ -440,11 +452,11 @@ mod tests { assert_ne!(scheme.to_cache_key(), base, "scheme must change the key"); let mut encoding = key(); - encoding.content_encoding = "br".to_string(); + encoding.accept_encoding = "br".to_string(); assert_ne!( encoding.to_cache_key(), base, - "content encoding must change the key; serving brotli to a gzip client \ + "accept encoding must change the key; serving brotli to a gzip client \ is a broken response" ); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 10f2a58fe..c8caa16af 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1284,6 +1284,49 @@ pub async fn buffer_publisher_response_async( } } +/// Builds the response served from a C2 hit. +/// +/// Every header is constructed here rather than replayed from the stored entry. The +/// publisher path rewrites `Cache-Control` and strips validators after the send, so a +/// replayed origin header would fight it; constructing them also means no origin header +/// can reach a second visitor through the cache, which makes the `Set-Cookie` privacy +/// net trivially safe rather than safe-by-audit. +/// +/// # Errors +/// +/// Returns an error if the stored metadata cannot be rendered as header values, which +/// would mean a corrupt entry. +/// +/// Spike-only, for the #1009 ESI validation. +fn build_cached_template_response( + entry: &crate::platform::TemplateEntry, +) -> Result, Report> { + let invalid = |what: &str| TrustedServerError::Proxy { + message: format!("cached template has an unusable {what}"), + }; + let mut response = Response::new(EdgeBody::from(entry.body.clone())); + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_str(&entry.metadata.content_type) + .change_context_lazy(|| invalid("content type"))?, + ); + // The encoding the origin actually chose, not the one keyed on. See + // `TemplateCacheKey::accept_encoding` for why those differ. + if !entry.metadata.content_encoding.is_empty() && entry.metadata.content_encoding != "identity" + { + response.headers_mut().insert( + header::CONTENT_ENCODING, + HeaderValue::from_str(&entry.metadata.content_encoding) + .change_context_lazy(|| invalid("content encoding"))?, + ); + } + response.headers_mut().insert( + header::CONTENT_LENGTH, + HeaderValue::from(entry.body.len() as u64), + ); + Ok(response) +} + /// Writes the transformed template to the shared cache, if the gate authorized it. /// /// The key's presence is the authorization: it is `Some` only when @@ -3032,23 +3075,44 @@ pub async fn handle_publisher_request( // legacy path never sets it. Either way it is an internal edge signal that // must not leak to publisher backends. req.headers_mut().remove("fastly-ssl"); - // Captured before the request is consumed: the C2 key identifies the origin - // document plus the request headers the origin varies on, and this is the last - // point where both are still in hand. + // The C2 key is built here, before the request is consumed, because every field + // is request-derived and this is the last point where the request is in hand. // - // Read from the request as forwarded, after `restrict_accept_encoding` — keying on - // what the client originally sent would key on a value the origin never saw. - let template_cache_url = target_uri.to_string(); - let template_cache_vary_values = settings - .creative_opportunities - .as_ref() - .map(CreativeOpportunitiesConfig::template_cache_vary) - .unwrap_or_else(|| VarySpec::new([])) - .values_from(|name| { - req.headers() - .get(name) + // Building it pre-fetch is what makes a lookup possible at all: a key that needed + // the origin's response could only ever authorize a store, never satisfy a read. + // + // Headers are read as forwarded, after `restrict_accept_encoding` — keying on what + // the client originally sent would key on a value the origin never saw. + let template_cache_key = (!matches!(assembly_mode, AssemblyMode::Inline)).then(|| { + crate::platform::TemplateCacheKey { + url: target_uri.to_string(), + request_host: request_host.to_string(), + request_scheme: request_scheme.to_string(), + assembly_mode, + vary_values: settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::template_cache_vary) + .unwrap_or_else(|| VarySpec::new([])) + .values_from(|name| { + req.headers() + .get(name) + .and_then(|value| value.to_str().ok()) + }), + accept_encoding: req + .headers() + .get(header::ACCEPT_ENCODING) .and_then(|value| value.to_str().ok()) - }); + .unwrap_or_default() + .to_string(), + // Changes whenever any JS module changes, so a bundle deploy invalidates + // stored templates without needing a purge. + integration_fingerprint: trusted_server_js::concatenated_hash( + &trusted_server_js::all_module_ids(), + ), + schema_version: crate::platform::TEMPLATE_SCHEMA_VERSION, + } + }); *req.uri_mut() = target_uri; req.headers_mut().insert( header::HOST, @@ -3057,6 +3121,32 @@ pub async fn handle_publisher_request( })?, ); + // C2 lookup, before the origin fetch — the whole point is to skip it. + // + // The gate that authorized the store was response-derived, so it cannot re-run + // here and does not need to: a template in the cache already passed it. What must + // re-run are the *request*-derived disqualifications, because they are properties + // of this request rather than of the stored bytes. An authenticated request must + // not be served a shared template even if that template is perfectly cacheable. + if let Some(key) = template_cache_key + .as_ref() + .filter(|_| !request_had_authorization && !request_had_cookie) + { + match services.template_cache().get(key).await { + Ok(entry) => { + log::debug!("c2_template_cache hit: {} bytes", entry.body.len()); + // Headers are constructed rather than replayed. The publisher path + // rewrites `Cache-Control` and strips validators *after* the send, so + // replaying stored origin headers would fight that — and constructing + // them means no origin header can leak through the cache. + return Ok(PublisherResponse::Buffered(build_cached_template_response( + &entry, + )?)); + } + Err(miss) => log::debug!("c2_template_cache miss: {miss}"), + } + } + // SSP requests are already racing through the platform HTTP client, so // origin TTFB tracks origin latency rather than the auction timeout. // @@ -3183,21 +3273,14 @@ pub async fn handle_publisher_request( .to_lowercase(); let route = classify_response_route(status, &content_type, &content_encoding, request_host); - // The shared-template cache gate. Evaluated here rather than earlier because the - // negotiated content encoding is part of the key: the pipeline pairs input encoding - // to output encoding, so a template stored as brotli must never be handed to a - // client that asked for gzip. + // The shared-template cache gate: it does not build the key, it authorizes storing + // the one built pre-fetch. Everything it checks is response-derived, which is + // exactly why it cannot run at lookup time — and why it does not need to. Anything + // already in the cache passed this gate on the way in. // - // A `Some` key is the store authorization. Under the default `Inline` mode the gate - // reports `InlineMode` and nothing is ever stored. - let template_cache_key = if matches!(assembly_mode, AssemblyMode::Inline) { - None - } else { - let key_vary = settings - .creative_opportunities - .as_ref() - .map(CreativeOpportunitiesConfig::template_cache_vary) - .unwrap_or_else(|| VarySpec::new([])); + // A surviving key is the store authorization. Under `Inline` there is no key to + // survive. + let template_cache_key = template_cache_key.filter(|_| { match c2_bypass_reason( assembly_mode, request_had_authorization, @@ -3205,31 +3288,22 @@ pub async fn handle_publisher_request( status, &content_type, response.headers(), - &key_vary, + &settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::template_cache_vary) + .unwrap_or_else(|| VarySpec::new([])), ) { Some(reason) => { log::debug!("c2_template_cache bypass: {reason}"); - None + false } None => { log::debug!("c2_template_cache eligible"); - Some(crate::platform::TemplateCacheKey { - url: template_cache_url, - request_host: request_host.to_string(), - request_scheme: request_scheme.to_string(), - assembly_mode, - vary_values: template_cache_vary_values, - content_encoding: content_encoding.clone(), - // Changes whenever any JS module changes, so a bundle deploy - // invalidates stored templates without needing a purge. - integration_fingerprint: trusted_server_js::concatenated_hash( - &trusted_server_js::all_module_ids(), - ), - schema_version: crate::platform::TEMPLATE_SCHEMA_VERSION, - }) + true } } - }; + }); match route { ResponseRoute::PassThrough => { @@ -5306,7 +5380,7 @@ mod tests { request_scheme: "https".to_string(), assembly_mode: AssemblyMode::Esi, vary_values: vec![], - content_encoding: "identity".to_string(), + accept_encoding: "identity".to_string(), integration_fingerprint: "fp".to_string(), schema_version: crate::platform::TEMPLATE_SCHEMA_VERSION, } @@ -5387,6 +5461,288 @@ mod tests { } } + mod c2_end_to_end_tests { + //! The chain, end to end: a second request for the same URL must be served from + //! the cache without touching the origin. Everything else in this file tests a + //! link; this tests that they connect. + + use super::*; + use crate::platform::ClientInfo; + use crate::platform::test_support::{ + NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, StubHttpClient, + }; + use crate::test_support::tests::crate_test_settings_str; + use std::collections::HashMap; + + /// A working cache, unlike the recorder above — this one has to actually return + /// what it stored, or a hit proves nothing. + #[derive(Default)] + struct MemoryTemplateCache { + entries: Mutex>, + } + + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformTemplateCache for MemoryTemplateCache { + async fn get( + &self, + key: &crate::platform::TemplateCacheKey, + ) -> Result + { + self.entries + .lock() + .expect("should lock entries") + .get(&key.to_cache_key()) + .cloned() + .ok_or(crate::platform::TemplateCacheMiss::NotFound) + } + + async fn put( + &self, + key: &crate::platform::TemplateCacheKey, + metadata: &crate::platform::TemplateMetadata, + body: Vec, + ) -> Result<(), crate::platform::TemplateCacheError> { + self.entries.lock().expect("should lock entries").insert( + key.to_cache_key(), + crate::platform::TemplateEntry { + metadata: metadata.clone(), + body, + }, + ); + Ok(()) + } + + async fn purge_all(&self) -> Result<(), crate::platform::TemplateCacheError> { + self.entries.lock().expect("should lock entries").clear(); + Ok(()) + } + } + + fn settings_with_mode(mode: &str) -> Settings { + let toml = format!( + "{}\n[creative_opportunities]\ngam_network_id = \"99999\"\n\ + assembly_mode = \"{mode}\"\n", + crate_test_settings_str() + ); + let mut settings = + Settings::from_toml(&toml).expect("should parse settings with an assembly mode"); + // Mirrors `create_test_settings`; the integration registry refuses to build + // without it. + settings.proxy.allowed_domains = + vec!["*.example".to_string(), "*.example.com".to_string()]; + settings + } + + fn services( + http_client: Arc, + cache: Arc, + ) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(http_client) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .template_cache(cache) + .build() + } + + /// Shareable HTML: no `Set-Cookie`, no `Vary`, a public `Cache-Control`. Every + /// condition the gate checks is satisfied, so a bypass here would be a bug in + /// the wiring rather than in the fixture. + fn queue_shareable_html(stub: &StubHttpClient) { + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); + } + + fn navigation_request() -> Request { + HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build navigation request") + } + + /// Runs the full request, **including the finalizer**. + /// + /// The finalizer is not optional here: the store happens once the transform has + /// produced every byte, so a test that stopped at `handle_publisher_request` + /// would never populate the cache and a hit could not be proven. + async fn run( + settings: &Arc, + services: &RuntimeServices, + request: Request, + ) -> Response { + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let registry = + IntegrationRegistry::new(settings).expect("should create integration registry"); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let publisher_response = handle_publisher_request( + settings, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + ) + .await + .expect("should proxy publisher request"); + + publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(settings), + ®istry, + orchestrator, + services.clone(), + ) + .await + .expect("should finalize publisher response") + } + + fn body_of(response: Response) -> Vec { + response + .into_body() + .into_bytes() + .expect("a shared-mode response is buffered, so its bytes are in hand") + .to_vec() + } + + #[tokio::test] + async fn a_second_request_is_served_from_the_cache_without_touching_the_origin() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + + // Only one origin response is queued. If the second request reached the + // origin it would find the queue empty, so this fixture is itself part of + // the assertion. + queue_shareable_html(&stub); + + let first = body_of(run(&settings, &services, navigation_request()).await); + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the cold request must fetch the origin" + ); + + let second = body_of(run(&settings, &services, navigation_request()).await); + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the warm request must not fetch the origin — that saving is the point" + ); + assert_eq!( + second, first, + "the cached template must be byte-identical to what was stored" + ); + } + + #[tokio::test] + async fn inline_mode_never_reads_or_writes_the_cache() { + // The shipped path. If this ever cached, per-user ad state would be shared + // between visitors — the exact failure the whole design exists to avoid. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("inline")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + let _ = run(&settings, &services, navigation_request()).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "inline must fetch the origin every time" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "inline must never write a shared template" + ); + } + + #[tokio::test] + async fn an_authenticated_request_is_not_served_a_shared_template() { + // The stored template is perfectly cacheable; this request is not entitled + // to it. The store gate cannot express that, because it is a property of + // the reader rather than of the bytes — which is why the lookup re-checks. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + assert_eq!( + cache.entries.lock().expect("should lock entries").len(), + 1, + "the cold request should have populated the cache" + ); + + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let authenticated = HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .body(EdgeBody::empty()) + .expect("should build authenticated request"); + let _ = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + authenticated, + ) + .await + .expect("should proxy publisher request"); + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "an authenticated request must reach the origin rather than read a \ + shared template" + ); + } + } + mod c2_gate_tests { //! `cache::core` stores whatever it is handed and rejects nothing, so every //! one of these conditions is the caller's to enforce. Each is a leak vector From fe0658d167c9a33bf6651820bd1e30856995b8bb Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 11:57:52 +0530 Subject: [PATCH 194/395] Execute the Core Cache join the publisher actually uses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The publisher tests use an in-memory double and the Fastly tests call the concrete type, so the seam between them was only ever type-checked: the publisher reaches the cache as a dyn PlatformTemplateCache behind RuntimeServices, which is what app.rs wires and what neither suite executed. Now round-tripped through the trait object under Viceroy against the real Core Cache. Records Task 3 as complete in the plan and the findings, including the three problems that only appeared once the code had to run — the Vary ordering, the encoding the origin chooses versus the one it is offered, and streaming not collecting the bytes a store needs. None were visible in the plan or in review, which is the same pattern as the earlier review findings arriving one layer down. Docs build verified, not just formatted. --- .../src/template_cache.rs | 19 ++++++++ .../2026-08-08-1009-measurement-findings.md | 33 ++++++++++++++ .../2026-08-10-1009-esi-validation-spike.md | 44 +++++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs index a7c2ceae8..3a32a617e 100644 --- a/crates/trusted-server-adapter-fastly/src/template_cache.rs +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -307,6 +307,25 @@ mod tests { ); } + #[test] + fn the_cache_round_trips_through_the_platform_trait_object() { + // Every other test here calls `FastlyTemplateCache` concretely. The publisher + // never does — it reaches the cache as a `dyn PlatformTemplateCache` behind + // `RuntimeServices`. That join is what `app.rs` wires, and until this test it + // was only type-checked, never executed. + let cache: std::sync::Arc = std::sync::Arc::new(cache()); + let key = key("https://example.com/via-trait-object"); + let body = b"template".to_vec(); + + run(cache.put(&key, &metadata_for(&body), body.clone())).expect("should store"); + + assert_eq!( + run(cache.get(&key)).expect("should read back").body, + body, + "the trait object must reach the same Core Cache the concrete type does" + ); + } + #[test] fn a_length_mismatch_is_refused_at_write_rather_than_stored() { // Storing metadata whose length disagrees with the body would make every diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md index 2d92512f9..ea58c61b0 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -350,6 +350,39 @@ Do not proceed to Task 3 Step 4 (actual C2 read/write) or expose `AssemblyMode` test or staging traffic until the full-document byte-identity test exists. The three fixes above close the known holes; that test is what would catch the next one. +## Task 3 complete — the C2 cache engages end to end + +`2db10639` (store), `2a2e6c6a` (lookup), plus `b688d667`/`577eb85a` for the `Vary` +handling. A second request for the same URL is now served without touching the origin, +byte-identical to what was stored. + +**Three problems only appeared once the code had to run**, none of them visible in the +plan or in review: + +1. **The key needed the origin's `Vary`, but a lookup precedes the fetch.** Resolved with + an operator-stated list plus a post-response drift guard that refuses to store under a + key that missed something. Spike-grade: a two-phase lookup is the correct answer and + doubles the lookups. +2. **The key carried the encoding the _origin_ chose**, which also does not exist at + lookup time — storing under `br`, looking up under `gzip, br`, a cache that never hits. + Now keyed on what was sent to the origin. +3. **Storing needs every transformed byte; streaming does not collect them.** Shared modes + take the buffered finalizer, branching on the store authorization rather than the + assembly mode, so `Inline` cannot reach it. + +Each was a case where the design read as complete and the implementation had a hole in +it. That is the same pattern as the three review findings above, arriving one layer down. + +**Verified by mutation, not just by green tests.** Disabling the lookup fails the hit +test, so the hit is the cache answering rather than the fixture answering twice; dropping +the `Authorization` re-check fails the authenticated test; reading only the first `Vary` +header value, and disabling the drift guard, each fail their own tests. The reviewer's +gate above was satisfied first: the byte-identity tests it demanded exist and were +themselves mutation-checked. + +**Still not deployable.** `ClientFill` and `Esi` render a template with a hole and +nothing filling it — Task 4 and Task 5. A cache that works is necessary, not sufficient. + ## Step B — consumers of TS's own response headers Not yet run. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index 9faa63914..c1f9cf1f3 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -614,6 +614,50 @@ cargo fmt --all -- --check && cargo clippy-fastly `ClientFill` must work on all four adapters. `Esi` is Fastly-only and must not break the others' compilation. +- [x] **Step 6: the call site — DONE.** `2db10639` (store), `2a2e6c6a` (lookup). + +The cache now engages end to end: a second request for the same URL is served without +touching the origin, and is byte-identical to what was stored. Verified by mutation — +disabling the lookup fails the hit test, so the hit is the cache answering rather than +the fixture answering twice. + +**Wiring the lookup corrected the key.** It carried the content encoding the _origin_ +chose, which does not exist at lookup time. That meant storing under `br` and looking up +under `gzip, br` — a cache that never hits. The field is now the `Accept-Encoding` sent +to the origin. Sound because negotiation is a function of what the origin was offered, +so identical offers yield identical choices; the chosen encoding stays in the metadata +and is what the served response declares. + +That made every key field request-derived, so **the key is built before the fetch** and +the response gate only authorizes storing it. A key that needed the response could only +ever authorize a store, never satisfy a read. + +**The lookup re-checks the request-derived disqualifications, and only those.** The +store gate is response-derived and cannot re-run, but need not: anything in the cache +passed it on the way in. What must re-run are properties of the _reader_ rather than of +the bytes — an authenticated request must not be served a shared template even when that +template is perfectly cacheable. + +**Shared modes take the buffered finalizer.** Storing needs every transformed byte and +streaming does not collect them. The branch keys on the store authorization rather than +on the assembly mode, so `Inline` never reaches it and the spike cannot regress the +shipped path by construction. A C2 _miss_ therefore buffers — the right trade, since a +miss is already paying an origin fetch and a full transform, and what the spike measures +is the hit, where there is no origin fetch to stream from at all. + +Every response header on a hit is constructed, never replayed, so no origin header can +reach a second visitor through the cache. + +The publisher tests use an in-memory cache double, so they prove the wiring rather than +the backing. The join they leave untested is the one `app.rs` makes: the publisher +reaches the cache as a `dyn PlatformTemplateCache` behind `RuntimeServices`, never as +the concrete type the Fastly tests exercise. That join is now executed under Viceroy +against the real Core Cache rather than only type-checked. + +**What this does not establish.** `ClientFill` and `Esi` still render a template with a +hole and nothing filling it. Task 4 and Task 5 remain the blockers on anything +deployable — a cache that works is necessary, not sufficient. + --- ## Task 4: Arm A2 — client-fill From 06864314d41db8ace9082b015ea857cbf64926a7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 12:24:53 +0530 Subject: [PATCH 195/395] Give the ESI seam a fragment to include MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Esi arm emitted nothing at because pointing an esi:include at /_ts/page-bids would splice raw JSON where an executable script belongs. This adds the script form and the marker that uses it. A format on the existing endpoint rather than a second path. Both forms carry the same data behind the same cross-site gate, so a new path would have duplicated that gate, the deprecation alias and the private, no-store header across four adapter routers for a difference in wrapping. As a format it is reachable on every adapter with no routing change at all. An unknown format is a 400, not a fall back to JSON. Defaulting would make a typo in an esi:include return 200 with a broken page and nothing in the logs pointing at the cause — the precise failure that kept this arm dark. The fragment reuses build_bids_script rather than formatting its own. If the two diverged, the A1-vs-A3 comparison would be measuring two script shapes instead of two delivery mechanisms and its number would mean nothing. Slots are not included: under a shared mode the head seam emits no tsjs.adSlots and the template already carries the slot markup, so the fragment supplies only what could not be shared. The marker carries no path. It is baked into a shared template, so every byte in it is a byte every reader of that template receives; the adapter's include dispatcher will supply the path from the live request. That also keeps a URL out of the cached bytes, so there is no escaping question at the seam. An existing test caught a real inconsistency this exposed. The root-auction gate asserted that dispatch usefulness tracks whether the seam emits bytes — true only while Esi emitted none. Under a Marker the seam emits bytes and reads nothing from ad_bids_state, because the fragment runs its own auction, so the old reading would have dispatched a root auction with no consumer: silent SSP spend, the exact waste that gate exists to prevent. The invariant now distinguishes emitting from consuming. Full gates green — fmt, six clippy targets, four adapter suites, 1889 core tests. Still open: the Fastly ESI processor is not wired, so the include is emitted and never resolved. Task 5. --- crates/trusted-server-core/src/publisher.rs | 288 +++++++++++++++++--- 1 file changed, 249 insertions(+), 39 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index c8caa16af..7d8001249 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -992,6 +992,12 @@ pub(crate) fn root_auction_is_useful(mode: AssemblyMode) -> bool { } } +/// The `esi:include` emitted at the `` seam under [`AssemblyMode::Esi`]. +/// +/// No `path` query parameter: see [`body_close_injection`]. The adapter's include +/// dispatcher appends it from the live request. +pub const ESI_BIDS_INCLUDE: &str = ""; + /// What the `` seam should inject, given the assembly mode. /// /// Explicit rather than inferred. The previous shape read @@ -999,12 +1005,16 @@ pub(crate) fn root_auction_is_useful(mode: AssemblyMode) -> bool { /// two independent decisions: once [`template_ad_slots_script`] stopped emitting a /// head script under a shared mode, body-close injection stopped with it. /// -/// `Esi` returns [`BodyCloseInjection::None`] for now rather than a placeholder -/// marker. The marker must point at a dedicated fragment endpoint returning an -/// executable script — `/_ts/page-bids` returns JSON, and ESI splices fragment -/// bytes verbatim, so aiming at it would put raw JSON where a script belongs. -/// That endpoint does not exist yet, and emitting a marker with nothing behind it -/// would be worse than emitting nothing. +/// `Esi` emits an `esi:include` pointing at the page-bids endpoint's **fragment** +/// format, which returns the same executable `"; + + fn template_with_include() -> String { + format!("
{ESI_BIDS_INCLUDE}") + } + + #[test] + fn the_seams_own_marker_is_resolved() { + // Deliberately built from `ESI_BIDS_INCLUDE` rather than a hand-written + // include. The two live in different crates, and a test that wrote its own + // marker would keep passing after the seam's changed shape stopped parsing. + let assembled = assemble(&template_with_include(), FRAGMENT).expect("should assemble"); + + assert!( + assembled.contains(FRAGMENT), + "the fragment must reach the document: {assembled}" + ); + assert!( + !assembled.contains("esi:include"), + "no unresolved include may survive: {assembled}" + ); + } + + #[test] + fn the_fragment_lands_where_the_marker_was() { + // Position matters: the script reads slots defined earlier in the document, so + // an assembler that appended instead of substituting would produce a page that + // parses and does nothing. + let assembled = assemble(&template_with_include(), FRAGMENT).expect("should assemble"); + + let slot = assembled.find("id=\"slot\"").expect("slot should survive"); + let script = assembled + .find(FRAGMENT) + .expect("fragment should be present"); + let body_close = assembled + .find("") + .expect("body close should survive"); + + assert!(slot < script, "the fragment must follow the slot markup"); + assert!(script < body_close, "the fragment must precede ``"); + } + + #[test] + fn a_document_without_an_include_is_returned_unchanged() { + // Inline mode's documents pass through this path only if something is + // misrouted, and a mangled document would be a far worse failure than a no-op. + let plain = "

no includes here

"; + + assert_eq!( + assemble(plain, FRAGMENT).expect("should assemble"), + plain, + "a template with nothing to splice must be byte-identical" + ); + } + + #[test] + fn an_empty_fragment_still_removes_the_marker() { + // The empty-bids case is normal, not exceptional: an auction that returned + // nothing still has to produce a document with no `esi:include` left in it, or + // the browser renders the raw tag as text. + let assembled = assemble(&template_with_include(), "").expect("should assemble"); + + assert!( + !assembled.contains("esi:include"), + "an empty fragment must still consume the marker: {assembled}" + ); + assert!(assembled.contains(""), "the document must survive"); + } + + #[test] + fn script_bearing_fragments_are_spliced_verbatim() { + // ESI substitutes bytes without escaping, which is exactly why the fragment + // endpoint must return markup rather than JSON. This pins that behaviour, since + // an `esi` release that started escaping would silently turn every fragment + // into visible text. + let fragment = ""; + let assembled = assemble(&template_with_include(), fragment).expect("should assemble"); + + assert!( + assembled.contains(fragment), + "the fragment must be spliced verbatim: {assembled}" + ); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 603ffdd9b..5ffb7226a 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -29,6 +29,7 @@ mod app; mod backend; mod compat; mod ec_kv; +mod esi_assembly; mod logging; mod management_api; mod middleware; From 0597f54e36c6354b400c94a0e13267b752ff897f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 12:41:15 +0530 Subject: [PATCH 197/395] State the ESI processor's safety settings instead of inheriting them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assembler used Configuration::default(). The plan's Task 5 Step 2 says explicitly not to, and reading the crate showed why that instruction exists. is_includes_cacheable defaults to true. A fragment here carries one visitor's bids, so letting the ESI layer cache it serves those bids to the next visitor — the exact per-user leak this whole design exists to prevent, arriving silently on a cache hit. That default fails open, in a pre-1.0 crate whose defaults can move in a patch release. Every safety-relevant field is now stated rather than inherited: - Fragment caching off, and includes_force_ttl left unset — it caches everything, ignoring private, no-store and Set-Cookie alike. - default_dca None and inherit_parent_dca false, so fragment bytes are never re-parsed as ESI. The fragment is a script built from auction data; parsing it as ESI would let bid content act as markup instructions. - max_include_depth 1. One include, no nesting. A template asking for more is not one this arm built. - Rendered caching and edge_control off. The publisher path sets private, no-store before any body byte is written, and headers cannot change once streaming starts on this adapter, so a Cache-Control computed from include TTLs would contradict it — and the contradiction would favour caching. Four tests assert the configuration rather than trusting it, plus one that proves the behaviour rather than the flag: a fragment containing its own esi:include is spliced as text, not dispatched, so auction data cannot drive fragment requests. Full gates green — fmt, six clippy targets, four adapter suites, 133 Fastly adapter tests. --- .../src/esi_assembly.rs | 101 +++++++++++++++++- 1 file changed, 99 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs index 85e6b25b3..a3282b98a 100644 --- a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs +++ b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs @@ -21,11 +21,53 @@ // and satisfied in the binary, and no single attribute can be both. #![allow(dead_code)] -use esi::{Configuration, PendingFragmentContent, Processor}; +use esi::{CacheConfig, Configuration, DcaMode, PendingFragmentContent, Processor}; use fastly::Response; use fastly::http::StatusCode; use std::io::Cursor; +/// The processor configuration, with every safety-relevant field stated. +/// +/// Not `Configuration::default()`. Two of these settings fail **open**, and this is a +/// pre-1.0 crate whose defaults can move in a patch release — a comment saying "the +/// default is already what we want" would be an assumption rechecked by nobody. +/// +/// The two that matter: +/// +/// - **`is_includes_cacheable` defaults to `true`.** Fragments here carry one visitor's +/// bids. Letting the ESI layer cache them is precisely the per-user leak this whole +/// design exists to prevent, and it would happen silently on a cache hit. +/// - **`default_dca` / `inherit_parent_dca`** decide whether fragment bytes are +/// re-parsed as ESI. Our fragment is a `"; + let assembled = assemble(&template_with_include(), fragment).expect("should assemble"); + + assert!( + assembled.contains("/evil"), + "the inner tag must survive as text rather than being resolved: {assembled}" + ); + } + #[test] fn script_bearing_fragments_are_spliced_verbatim() { // ESI substitutes bytes without escaping, which is exactly why the fragment From 2617ecc248832d24aa6c3a4b4ed7a0cf60e813f0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 12:42:21 +0530 Subject: [PATCH 198/395] Record that ESI's mechanism is proven and its defaults are not safe Two findings worth keeping out of commit messages alone. The async/sync obstacle is dissolved rather than worked around: PendingFragmentContent::CompletedRequest means the dispatcher performs no I/O, so the plan no longer needs the self-referencing backend it assumed. And the plan's "call the setters, do not trust the defaults" instruction turned out to be load-bearing: is_includes_cacheable defaults to true, which caches per-user bid fragments and serves them to the next visitor. Docs build verified, not just formatted. --- .../2026-08-10-1009-esi-validation-spike.md | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index c1f9cf1f3..a62006566 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -686,6 +686,37 @@ for the silent-empty-bids trap, which applies in full. ## Task 5: Arm A3 — ESI at the edge +- [x] **Step 0: the mechanism works — DONE.** `9539061e`, hardened in `0597f54e`. + +Verified under Viceroy with the real `esi` 0.7 crate rather than argued from docs: a +template carrying the `` seam's own `esi:include` comes back with the fragment +spliced in its place and no unresolved tag left. + +**The async/sync obstacle is dissolved, not worked around.** `esi`'s fragment dispatcher +is synchronous and this codebase's fragment producer is `async`; calling one from the +other means a nested executor, which panics. +`PendingFragmentContent::CompletedRequest` lets the dispatcher hand back an +already-built response, so the caller resolves the fragment in the normal async flow and +the dispatcher performs **no I/O at all** — no subrequest, no backend, no self-call, +nothing for Viceroy to stub. That also removes the need for a self-referencing backend +this plan would otherwise have required. + +**Step 2's instruction was right, and reading the crate showed why.** +`CacheConfig::is_includes_cacheable` defaults to **`true`**. A fragment carries one +visitor's bids, so the default caches per-user data and serves it to the next visitor — +silently, on a hit. `includes_force_ttl` is worse where set: it caches everything, +ignoring `private`, `no-store` and `Set-Cookie` alike. Both now stated explicitly, along +with `default_dca`/`inherit_parent_dca` (fragment bytes are data, never re-parsed as +ESI), `max_include_depth = 1`, and rendered caching / `edge_control` off because the +publisher path owns those headers. + +Nine tests. Four assert the configuration; the rest assert behaviour, including that a +fragment containing its own `esi:include` is spliced as text rather than dispatched, so +auction data cannot drive fragment requests. + +**What remains is the call site**, below. Emitting the include and resolving it are both +proven; connecting them is not done. + - [ ] **Step 1: Wire `process_stream`, not the wrappers** `process_response` and `process_response_streaming` consume `self` _and_ send the response From 0adb578ead28f7d3cad4c1e184f999a6b462bcd8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 12:58:13 +0530 Subject: [PATCH 199/395] Stop a C2 hit from serving a shared-cacheable per-user document MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cache hit returns before the origin fetch, and therefore before the point where the publisher path stamps private, no-store and strips validators. Nothing else set it, so a hit served HTML with no Cache-Control at all. That is not a safe default. HTML with no Cache-Control is heuristically cacheable by browsers and intermediaries, so an assembled per-user response was eligible to be stored and shared — the C3 the design forbids outright, reached by omission rather than by anything anyone wrote. The plan's Task 6 predicted exactly this class of miss. It says assert positively, because forbidding public, s-maxage and Surrogate-Control passes trivially when there is no Cache-Control to forbid. Checking for their absence would have reported this bug as safe. The hit path now stamps private, no-store first rather than last, and two tests assert it. One covers the returning visitor specifically: a first-visit response sets an EC cookie and the adapter's cookie-privacy net force-privatizes it regardless, so a test that only exercised first visits would pass on the backstop rather than on this code. A returning visitor sets no cookie, the net never fires, and this path is the only thing between the document and a shared cache. Verified by mutation: removing the stamp fails both new tests, with the hit and isolation tests still passing. Full gates green — fmt, six clippy targets, four adapter suites. --- crates/trusted-server-core/src/publisher.rs | 94 +++++++++++++++++++-- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 7d8001249..64ae53125 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1297,11 +1297,19 @@ pub async fn buffer_publisher_response_async( /// Builds the response served from a C2 hit. /// -/// Every header is constructed here rather than replayed from the stored entry. The -/// publisher path rewrites `Cache-Control` and strips validators after the send, so a -/// replayed origin header would fight it; constructing them also means no origin header -/// can reach a second visitor through the cache, which makes the `Set-Cookie` privacy -/// net trivially safe rather than safe-by-audit. +/// Every header is constructed here rather than replayed from the stored entry, so no +/// origin header can reach a second visitor through the cache. +/// +/// # Why this sets `private, no-store` itself +/// +/// A C2 hit returns **before** the origin fetch, and therefore before the point where +/// the publisher path stamps `private, no-store` and strips validators. Omitting it +/// here does not fall back to a safe default — it emits HTML with no `Cache-Control` at +/// all, which is heuristically cacheable by browsers and intermediaries. That is a +/// shared cache of an assembled per-user response: the C3 the design forbids outright. +/// +/// Asserting the absence of `public`/`s-maxage`/`Surrogate-Control` would not have +/// caught it. Nothing was present to forbid. /// /// # Errors /// @@ -1316,6 +1324,12 @@ fn build_cached_template_response( message: format!("cached template has an unusable {what}"), }; let mut response = Response::new(EdgeBody::from(entry.body.clone())); + // First, not last: see the note above. The assembled response is per-user even + // though the template it was built from is not. + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); response.headers_mut().insert( header::CONTENT_TYPE, HeaderValue::from_str(&entry.metadata.content_type) @@ -5867,6 +5881,76 @@ mod tests { ); } + fn header_of(response: &Response, name: header::HeaderName) -> Option<&str> { + response.headers().get(name).and_then(|v| v.to_str().ok()) + } + + #[tokio::test] + async fn a_cache_hit_is_never_shared_cacheable() { + // Asserted positively, because the obvious negative check does not work. + // Forbidding `public`, `s-maxage` and `Surrogate-Control` passes trivially + // on a response that carries no `Cache-Control` at all — and *that* is the + // real failure mode here, since a C2 hit returns before the point where the + // publisher path stamps the response private. HTML with no `Cache-Control` + // is heuristically cacheable, so "nothing to forbid" is not safety. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + + let _cold = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the second request must be a hit, or this asserts nothing" + ); + assert_eq!( + header_of(&warm, header::CACHE_CONTROL), + Some("private, no-store"), + "an assembled response is per-user even when its template is not" + ); + + // Validators would let a client revalidate into a shared copy, and the CDN + // directives would instruct an intermediary to store one outright. The + // origin fixture sends a `public, max-age=300` that must not survive. + for stripped in [header::ETAG, header::LAST_MODIFIED, header::EXPIRES] { + assert_eq!( + header_of(&warm, stripped.clone()), + None, + "{stripped} must not survive onto an assembled response" + ); + } + } + + #[tokio::test] + async fn a_returning_visitor_gets_the_same_privacy_headers() { + // The case with no backstop. A first-visit response sets an EC cookie, so + // the adapter's cookie-privacy net force-privatizes it regardless of what + // this path does. A returning visitor sets no cookie, so that net never + // fires and this path is the only thing standing between an assembled + // per-user document and a shared cache. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + + let _cold = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; + + assert!( + header_of(&warm, header::SET_COOKIE).is_none(), + "no cookie here means no privacy net, which is the point of this test" + ); + assert_eq!( + header_of(&warm, header::CACHE_CONTROL), + Some("private, no-store") + ); + } + #[tokio::test] async fn inline_mode_never_reads_or_writes_the_cache() { // The shipped path. If this ever cached, per-user ad state would be shared From b3ac59a6b58e17fc48816de8cb71e0508f1886c6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 13:39:57 +0530 Subject: [PATCH 200/395] Resolve the ESI include in the request path so ads render MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the gap between emitting a marker and filling it. Both were proven separately; nothing connected them, so a shared-mode page returned 200 with a literal esi:include in it — no ads, no error, and every monitor reporting success. A design correction first. Esi's root auction was gated off on the premise that the fragment would run its own auction via a real subrequest. That premise made the arm strictly worse: a self-referencing backend, two auction code paths, and two auctions per pageview. It is also not what esi requires — CompletedRequest satisfies an include from bytes already in hand. So the auction already in flight *is* the fragment, and root_auction_is_useful(Esi) is now true. ClientFill stays false; the browser fetches its own bids, so a root auction there genuinely has no consumer. Assembly sits behind a platform trait, like the template cache, because the only implementation uses a Fastly-only crate. The default is UnavailableTemplateAssembler, which refuses rather than passing the template through: returning it unchanged is the tempting default and it produces exactly the silent no-ads page this commit exists to prevent. Core's tests use a plain substitution instead, which is what one constant marker reduces to — and which shows the seam is portable even though the crate is not. Two call sites, deliberately not one. The miss path assembles after the transform; the hit path assembles after reading the cache. Keeping them separate is what makes the store-before-assemble ordering visible rather than implied. That ordering also revealed a bug on the hit path: it returned before the pipeline that normally collects the auction, so a hit dropped its in-flight auction — billing the SSPs for a result nobody read, the exact waste the dispatch gate exists to prevent, reappearing on the one path that skips the pipeline. The hit path now collects and assembles. Two tests carry the load. One asserts the marker never reaches the browser on either path, checking both because only one call site running would still look like success on the other. The other asserts the cached template holds the marker and never a bids script. The second one earns its place: swapping store and assemble fails it and nothing else. Every other test still passes, including the marker test, because the served page looks correct — one visitor's bids would simply be in a cache shared with the next. Verified by running that mutation. Full gates green — fmt, six clippy targets, four adapter suites, 1887 core tests, 133 Fastly adapter tests. --- .../trusted-server-adapter-fastly/src/app.rs | 1 + .../src/esi_assembly.rs | 18 +- .../trusted-server-core/src/platform/mod.rs | 6 +- .../src/platform/template_assembly.rs | 87 ++++ .../trusted-server-core/src/platform/types.rs | 42 ++ crates/trusted-server-core/src/publisher.rs | 405 ++++++++++++------ 6 files changed, 414 insertions(+), 145 deletions(-) create mode 100644 crates/trusted-server-core/src/platform/template_assembly.rs diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index c85e24670..dd89887a4 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -259,6 +259,7 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime // Spike-only (#1009). Constructed unconditionally, but only read when the // assembly mode is a shared-template one — which defaults to Inline, so this // is inert until an operator opts in. + .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) .template_cache(Arc::new(crate::template_cache::FastlyTemplateCache::new( crate::template_cache::TEMPLATE_CACHE_TTL, ))) diff --git a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs index a3282b98a..9d06765b0 100644 --- a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs +++ b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs @@ -15,16 +15,11 @@ //! //! Spike-only. Remove with the spike. -// Not yet reachable from the request path: resolving the fragment means running the -// auction, which is the next step (the spike plan's Task 5). The tests below do -// exercise it, so `expect` is wrong here — it would be unfulfilled under `cfg(test)` -// and satisfied in the binary, and no single attribute can be both. -#![allow(dead_code)] - use esi::{CacheConfig, Configuration, DcaMode, PendingFragmentContent, Processor}; use fastly::Response; use fastly::http::StatusCode; use std::io::Cursor; +use trusted_server_core::platform::{PlatformTemplateAssembler, TemplateAssemblyError}; /// The processor configuration, with every safety-relevant field stated. /// @@ -131,6 +126,17 @@ pub fn assemble(template: &str, fragment: &str) -> Result Result { + assemble(template, fragment).map_err(|e| TemplateAssemblyError::Failed { + message: e.to_string(), + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 2ff2fb06a..8fae8a64a 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -36,7 +36,8 @@ mod error; mod http; mod image_optimizer; mod kv; -pub mod template_cache; +pub mod template_assembly; +mod template_cache; #[cfg(test)] pub(crate) mod test_support; mod traits; @@ -53,6 +54,9 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; +pub use template_assembly::{ + PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, +}; pub use template_cache::{ PlatformTemplateCache, TEMPLATE_SCHEMA_VERSION, TemplateCacheError, TemplateCacheKey, TemplateCacheMiss, TemplateEntry, TemplateMetadata, UnavailableTemplateCache, VarySpec, diff --git a/crates/trusted-server-core/src/platform/template_assembly.rs b/crates/trusted-server-core/src/platform/template_assembly.rs new file mode 100644 index 000000000..cdf3033d9 --- /dev/null +++ b/crates/trusted-server-core/src/platform/template_assembly.rs @@ -0,0 +1,87 @@ +//! Edge assembly: turn a shared template plus a per-user fragment into a document. +//! +//! Kept behind a trait for the same reason as the template cache: the only +//! implementation that exists uses a Fastly-only crate, and core must stay portable. +//! Adapters without one get [`UnavailableTemplateAssembler`], which refuses rather than +//! guessing. +//! +//! **Ordering this module exists to protect.** The template is stored *before* assembly +//! and assembled *after* — never the reverse. Storing post-assembly would put one +//! visitor's bids in a cache shared with the next, which is the C3 the design forbids. +//! Splitting store from assemble into two call sites is what makes that ordering +//! visible instead of implicit. +//! +//! Spike-only, for the #1009 ESI validation. + +use core::fmt; + +/// Why assembly could not produce a document. +#[derive(Debug, derive_more::Display)] +pub enum TemplateAssemblyError { + /// The adapter has no assembler. + /// + /// Not a failure to be papered over: reaching here means a shared-template mode is + /// configured on an adapter that cannot serve one, and the honest response is an + /// error rather than a page with an unresolved marker in it. + #[display("this adapter cannot assemble shared templates")] + Unsupported, + /// The assembler ran and failed. + #[display("template assembly failed: {message}")] + Failed { + /// What the underlying assembler reported. + message: String, + }, +} + +impl core::error::Error for TemplateAssemblyError {} + +/// Splices a per-user fragment into a shared template. +pub trait PlatformTemplateAssembler: Send + Sync { + /// Produce the document served to this visitor. + /// + /// # Errors + /// + /// Returns [`TemplateAssemblyError::Unsupported`] when the adapter has no + /// assembler, or [`TemplateAssemblyError::Failed`] when the template could not be + /// processed. + fn assemble(&self, template: &str, fragment: &str) -> Result; +} + +impl fmt::Debug for dyn PlatformTemplateAssembler { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformTemplateAssembler") + } +} + +/// The default: no assembler. +/// +/// Refuses rather than returning the template unchanged. Returning it unchanged would +/// serve a page whose ad markup is a literal `esi:include` — a page that looks like it +/// worked, renders no ads, and reports no error. +#[derive(Debug, Default, Clone, Copy)] +pub struct UnavailableTemplateAssembler; + +impl PlatformTemplateAssembler for UnavailableTemplateAssembler { + fn assemble(&self, _template: &str, _fragment: &str) -> Result { + Err(TemplateAssemblyError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_null_assembler_refuses_rather_than_passing_the_template_through() { + // Passing it through is the tempting default and the wrong one: the visitor + // gets a page with a raw `esi:include` in it, no ads, and no error anywhere. + let error = UnavailableTemplateAssembler + .assemble( + "", + "", + ) + .expect_err("an adapter with no assembler must refuse"); + + assert!(matches!(error, TemplateAssemblyError::Unsupported)); + } +} diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index a1e48b11c..21fd164d2 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -173,6 +173,11 @@ pub struct RuntimeServices { /// per request rather than failing. Spike-only; see /// [`crate::platform::template_cache`]. pub(crate) template_cache: Arc, + /// Edge assembler for shared templates. Defaults to + /// [`UnavailableTemplateAssembler`], which refuses rather than serving a document + /// with an unresolved marker in it. Spike-only; see + /// [`crate::platform::template_assembly`]. + pub(crate) template_assembler: Arc, /// Dynamic backend registration and name prediction. pub(crate) backend: Arc, /// Outbound HTTP client abstraction. @@ -234,6 +239,12 @@ impl RuntimeServices { &*self.template_cache } + /// The edge template assembler. Spike-only. + #[must_use] + pub fn template_assembler(&self) -> &dyn super::PlatformTemplateAssembler { + &*self.template_assembler + } + /// Returns the dynamic backend service. #[must_use] pub fn backend(&self) -> &dyn PlatformBackend { @@ -294,6 +305,20 @@ impl RuntimeServices { ..self } } + + /// Returns a clone of this instance with the template assembler replaced. + /// + /// Spike-only (#1009). + #[must_use] + pub fn with_template_assembler( + self, + assembler: Arc, + ) -> Self { + Self { + template_assembler: assembler, + ..self + } + } } impl fmt::Debug for RuntimeServices { @@ -313,6 +338,7 @@ pub struct RuntimeServicesBuilder { secret_store: Option>, kv_store: Option>, template_cache: Option>, + template_assembler: Option>, backend: Option>, http_client: Option>, geo: Option>, @@ -327,6 +353,7 @@ impl RuntimeServicesBuilder { secret_store: None, kv_store: None, template_cache: None, + template_assembler: None, backend: None, http_client: None, geo: None, @@ -356,6 +383,16 @@ impl RuntimeServicesBuilder { self } + /// Set the edge template assembler. Spike-only. + #[must_use] + pub fn template_assembler( + mut self, + assembler: Arc, + ) -> Self { + self.template_assembler = Some(assembler); + self + } + /// Set the KV store implementation. #[must_use] pub fn kv_store(mut self, kv_store: Arc) -> Self { @@ -423,6 +460,11 @@ impl RuntimeServicesBuilder { template_cache: self .template_cache .unwrap_or_else(|| Arc::new(super::UnavailableTemplateCache)), + // Defaulted to a refusal rather than to a pass-through: an adapter with no + // assembler must not serve a template with an unresolved marker in it. + template_assembler: self + .template_assembler + .unwrap_or_else(|| Arc::new(super::UnavailableTemplateAssembler)), backend: self .backend .expect("should set backend before building RuntimeServices"), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 64ae53125..8896522d5 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -987,8 +987,17 @@ pub(crate) fn template_gpt_diagnostics( pub(crate) fn root_auction_is_useful(mode: AssemblyMode) -> bool { match mode { AssemblyMode::Inline => true, - // The fragment path runs its own auction; see the spike plan's Task 4. - AssemblyMode::ClientFill | AssemblyMode::Esi => false, + // The browser fetches its own bids after load, so a root auction here would be + // a second one nothing reads. + AssemblyMode::ClientFill => false, + // Consumed by edge assembly rather than by a seam. An earlier revision returned + // `false` here on the premise that the fragment would run its own auction via a + // real subrequest. That premise made the arm strictly worse — a self-referencing + // backend, two auction paths, and two auctions per pageview — and it is not what + // `esi` requires: `PendingFragmentContent::CompletedRequest` lets the include be + // satisfied from bytes already in hand. So the auction already in flight *is* + // the fragment, and it is very much consumed. + AssemblyMode::Esi => true, } } @@ -1280,7 +1289,11 @@ pub async fn buffer_publisher_response_async( ) .await?; let bytes = output.into_inner(); + // Store first, assemble second — never the reverse. The stored bytes are + // shared between visitors; the assembled ones carry this visitor's bids. + // Swapping these two lines is the C3 leak. store_template_if_authorized(services, &mut params, &bytes).await; + let bytes = assemble_if_shared(services, settings, ¶ms, bytes)?; response.headers_mut().insert( http::header::CONTENT_LENGTH, http::HeaderValue::from(bytes.len() as u64), @@ -1295,6 +1308,96 @@ pub async fn buffer_publisher_response_async( } } +/// Resolves the `` marker into this visitor's bids, if the mode assembles. +/// +/// Called *after* [`store_template_if_authorized`], never before: what is stored must +/// be the template every visitor shares, and what is returned must be this visitor's +/// document. Two call sites rather than one so that ordering is visible rather than +/// implied. +/// +/// # Errors +/// +/// Returns an error if the adapter has no assembler or if assembly fails. Deliberately +/// fatal rather than falling back to the unassembled template: that template contains a +/// literal `esi:include`, so serving it would render no ads, report no error, and look +/// to every monitor like a page that worked. +fn assemble_if_shared( + services: &RuntimeServices, + settings: &Settings, + params: &OwnedProcessResponseParams, + bytes: Vec, +) -> Result, Report> { + let assembly_mode = settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::assembly_mode) + .unwrap_or_default(); + if !matches!(assembly_mode, AssemblyMode::Esi) { + return Ok(bytes); + } + + let template = String::from_utf8(bytes).change_context(TrustedServerError::Proxy { + message: "shared template is not valid UTF-8, so it cannot be assembled".to_string(), + })?; + + // The auction already in flight is the fragment. `body_close_injection` emitted a + // constant marker into the template precisely so this substitution — not a + // subrequest — is what fills it. + let fragment = params + .ad_bids_state + .lock() + .expect("should lock bid state") + .clone() + .unwrap_or_else(build_empty_bids_script); + + services + .template_assembler() + .assemble(&template, &fragment) + .map(String::into_bytes) + .change_context(TrustedServerError::Proxy { + message: "failed to assemble the shared template".to_string(), + }) +} + +/// Collects the in-flight auction and assembles the cached template with its bids. +/// +/// A C2 hit skips the origin fetch, and with it the streaming pipeline that normally +/// collects the auction and fills the `` seam. Both still have to happen — the +/// auction was dispatched before the lookup and is already billing the SSPs. +/// +/// # Errors +/// +/// Returns an error if the cached bytes are not UTF-8, or if assembly fails. +async fn collect_and_assemble_cached_template( + entry: &crate::platform::TemplateEntry, + dispatched: Option, + telemetry: AuctionTelemetryCarry, + deps: &AuctionCollectDeps<'_>, +) -> Result, Report> { + if let Some(dispatched) = dispatched { + collect_stream_auction(dispatched, telemetry, deps).await; + } + + let template = core::str::from_utf8(&entry.body).change_context(TrustedServerError::Proxy { + message: "cached template is not valid UTF-8, so it cannot be assembled".to_string(), + })?; + + let fragment = deps + .ad_bids_state + .lock() + .expect("should lock bid state") + .clone() + .unwrap_or_else(build_empty_bids_script); + + deps.services + .template_assembler() + .assemble(template, &fragment) + .map(String::into_bytes) + .change_context(TrustedServerError::Proxy { + message: "failed to assemble the cached template".to_string(), + }) +} + /// Builds the response served from a C2 hit. /// /// Every header is constructed here rather than replayed from the stored entry, so no @@ -1319,11 +1422,13 @@ pub async fn buffer_publisher_response_async( /// Spike-only, for the #1009 ESI validation. fn build_cached_template_response( entry: &crate::platform::TemplateEntry, + assembled: Vec, ) -> Result, Report> { let invalid = |what: &str| TrustedServerError::Proxy { message: format!("cached template has an unusable {what}"), }; - let mut response = Response::new(EdgeBody::from(entry.body.clone())); + let assembled_len = assembled.len() as u64; + let mut response = Response::new(EdgeBody::from(assembled)); // First, not last: see the note above. The assembled response is per-user even // though the template it was built from is not. response.headers_mut().insert( @@ -1345,10 +1450,11 @@ fn build_cached_template_response( .change_context_lazy(|| invalid("content encoding"))?, ); } - response.headers_mut().insert( - header::CONTENT_LENGTH, - HeaderValue::from(entry.body.len() as u64), - ); + // The assembled length, not the template's: assembly substitutes the marker for a + // bids script, so the two differ on every request. + response + .headers_mut() + .insert(header::CONTENT_LENGTH, HeaderValue::from(assembled_len)); Ok(response) } @@ -2938,15 +3044,13 @@ pub async fn handle_publisher_request( let mut dispatched_auction = if matched_slots.is_empty() { None } else if !root_auction_is_useful(assembly_mode) { - // Shared-template modes inject nothing at the root: `template_ad_slots_script` - // and `body_close_injection` both return `None`. Dispatching here would send - // real SSP requests, hold the response for the full auction budget, and then - // discard the result with no error and no log — the silent-waste signature - // §5 of the design doc is entirely about. The fragment path runs its own - // auction; this one has no consumer. + // `ClientFill` injects nothing and the browser fetches its own bids, so + // dispatching here would send real SSP requests, hold the response for the full + // auction budget, and then discard the result with no error and no log — the + // silent-waste signature §5 of the design doc is entirely about. log::debug!( - "skipping root auction dispatch: assembly mode {assembly_mode:?} injects \ - nothing at the root" + "skipping root auction dispatch: assembly mode {assembly_mode:?} has no \ + consumer for the result" ); None } else { @@ -3160,12 +3264,34 @@ pub async fn handle_publisher_request( match services.template_cache().get(key).await { Ok(entry) => { log::debug!("c2_template_cache hit: {} bytes", entry.body.len()); + // The origin fetch is skipped, but the auction is not. It was dispatched + // above and is in flight; dropping it here would bill the SSPs for a + // result nobody reads — the silent waste the dispatch gate exists to + // prevent, reappearing on the one path that skips the pipeline which + // normally collects it. + let assembled = collect_and_assemble_cached_template( + &entry, + dispatched_auction.take(), + AuctionTelemetryCarry { + observation: auction_observation.take(), + auction_request: auction_request_for_telemetry.clone(), + }, + &AuctionCollectDeps { + price_granularity, + ad_bids_state: &ad_bids_state, + orchestrator: auction.orchestrator, + services, + settings, + request_origin: request_origin(request_scheme, request_host), + }, + ) + .await?; // Headers are constructed rather than replayed. The publisher path // rewrites `Cache-Control` and strips validators *after* the send, so // replaying stored origin headers would fight that — and constructing // them means no origin header can leak through the cache. return Ok(PublisherResponse::Buffered(build_cached_template_response( - &entry, + &entry, assembled, )?)); } Err(miss) => log::debug!("c2_template_cache miss: {miss}"), @@ -5320,152 +5446,64 @@ mod tests { mod root_auction_gate_tests { //! Guards the silent-waste failure mode: dispatching an auction whose result - //! nothing will consume. Under the shared modes both injection seams emit - //! nothing, so a dispatched root auction bills the SSPs, holds the response - //! for the full budget, and discards the result with no error and no log. + //! nothing will consume. A dispatched root auction bills the SSPs and holds the + //! response for the full budget, so discarding the result is real spend with no + //! error and no log. use super::*; use crate::creative_opportunities::AssemblyMode; #[test] - fn only_inline_has_a_consumer_for_a_root_auction() { + fn a_mode_dispatches_exactly_when_something_will_read_the_result() { + // Stated per-variant because the three modes consume the result in three + // different ways, and an earlier revision that derived this from the seam + // decision got `Esi` wrong twice in opposite directions. assert!( root_auction_is_useful(AssemblyMode::Inline), - "inline injects the auction result at ``" + "inline reads `ad_bids_state` at the `` seam" + ); + assert!( + root_auction_is_useful(AssemblyMode::Esi), + "esi consumes the result at edge assembly rather than at a seam" + ); + assert!( + !root_auction_is_useful(AssemblyMode::ClientFill), + "client-fill fetches its own bids after load, so a root auction here \ + would be a second one nothing reads" ); - for mode in [AssemblyMode::ClientFill, AssemblyMode::Esi] { - assert!( - !root_auction_is_useful(mode), - "{mode:?}: neither seam reads `ad_bids_state`, so a root auction has \ - no consumer" - ); - } - } - - #[test] - fn the_gate_agrees_with_the_injection_decisions() { - // The invariant is about *consuming* the root auction's result, not about - // emitting bytes. `InlineBids` reads `ad_bids_state`; a `Marker` is verbatim - // and reads nothing, because the fragment behind it runs its own auction. - // - // Distinguishing those is the whole point: an earlier revision equated - // "the seam emits something" with "the seam consumes the auction", which - // held only while `Esi` emitted nothing. The moment it emitted an - // `esi:include`, that reading would have dispatched a root auction whose - // result nothing reads — silent SSP spend, exactly the waste the gate - // exists to prevent. - for mode in [ - AssemblyMode::Inline, - AssemblyMode::ClientFill, - AssemblyMode::Esi, - ] { - let consumes_the_root_auction = matches!( - body_close_injection(mode, true), - BodyCloseInjection::InlineBids - ); - assert_eq!( - root_auction_is_useful(mode), - consumes_the_root_auction, - "{mode:?}: dispatch usefulness must track whether a seam reads the \ - root auction's result" - ); - } } #[test] - fn a_marker_seam_does_not_dispatch_a_root_auction() { - // The waste case, stated directly. `Esi` emits bytes at `` but reads - // nothing from `ad_bids_state`, so dispatching a root auction for it would - // buy SSP responses that are discarded. + fn consuming_the_result_is_not_the_same_as_emitting_bytes() { + // The distinction that made both earlier revisions wrong. `Esi` emits a + // `Marker` and reads nothing from `ad_bids_state` — so a gate derived from + // "does the seam emit" or from "does the seam read `ad_bids_state`" lands on + // the wrong answer. The result reaches the page through assembly, not + // through the seam. assert!(matches!( body_close_injection(AssemblyMode::Esi, true), BodyCloseInjection::Marker(_) )); - assert!( - !root_auction_is_useful(AssemblyMode::Esi), - "the fragment runs its own auction; a second one at the root is spend \ - with no consumer" - ); - } - } - - mod body_close_decision_tests { - //! The `` decision must not be inferred from the `` script. - //! - //! Coupling them is a live defect, not a hypothetical: gating the head seam - //! on template neutrality made `ad_slots_script` `None` under shared modes, - //! which silently disabled body-close injection too. These tests pin the two - //! decisions apart. - - use super::*; - use crate::creative_opportunities::AssemblyMode; - - #[test] - fn inline_injects_bids_only_when_the_head_script_is_present() { - assert_eq!( - body_close_injection(AssemblyMode::Inline, true), + assert_ne!( + body_close_injection(AssemblyMode::Esi, true), BodyCloseInjection::InlineBids, - "inline with matched slots should inject the auction result" - ); - assert_eq!( - body_close_injection(AssemblyMode::Inline, false), - BodyCloseInjection::None, - "inline without matched slots should leave the publisher's flow alone" - ); - } - - #[test] - fn shared_modes_do_not_depend_on_the_head_script() { - // The decision must be the same either way. Under a shared mode the head - // script is always absent, so a decision that read it would be - // accidentally correct here and wrong the moment that changes. - for mode in [AssemblyMode::ClientFill, AssemblyMode::Esi] { - assert_eq!( - body_close_injection(mode, true), - body_close_injection(mode, false), - "{mode:?}: body-close must not vary with head-script presence" - ); - } - } - - #[test] - fn client_fill_emits_nothing_because_the_browser_fetches_unprompted() { - assert_eq!( - body_close_injection(AssemblyMode::ClientFill, false), - BodyCloseInjection::None - ); - } - - #[test] - fn esi_emits_an_include_aimed_at_the_fragment_format() { - // Pointing at the default JSON form would splice raw JSON where an - // executable script belongs — the failure that kept this arm emitting - // nothing until the fragment format existed. - assert_eq!( - body_close_injection(AssemblyMode::Esi, false), - BodyCloseInjection::Marker(ESI_BIDS_INCLUDE.to_string()) + "the esi seam does not read the auction result" ); assert!( - ESI_BIDS_INCLUDE.contains("format=fragment"), - "the include must request the script form, not the default JSON" + root_auction_is_useful(AssemblyMode::Esi), + "and yet the auction is dispatched, because assembly reads it" ); } #[test] - fn the_esi_marker_is_identical_regardless_of_request() { - // The marker is baked into a *shared* template, so every byte in it is a - // byte every reader of that template receives. A per-request value here — - // a path, an id, a nonce — would be one visitor's data served to the next. + fn a_mode_that_injects_nothing_and_assembles_nothing_never_dispatches() { + // `ClientFill` is the one mode where both are true, and it is the case the + // silent-waste guard exists for. assert_eq!( - body_close_injection(AssemblyMode::Esi, true), - body_close_injection(AssemblyMode::Esi, false), - "the marker must not vary with request state" - ); - assert!( - !ESI_BIDS_INCLUDE.contains("path="), - "the path comes from the live request at include time, not from the \ - cached bytes" + body_close_injection(AssemblyMode::ClientFill, true), + BodyCloseInjection::None ); + assert!(!root_auction_is_useful(AssemblyMode::ClientFill)); } } @@ -5757,6 +5795,24 @@ mod tests { settings } + /// A plain substitution assembler. + /// + /// Core has no ESI crate — the Fastly adapter owns that. Substituting the marker + /// directly is what an `esi:include` with one constant marker reduces to, so + /// this exercises the seam faithfully without importing a Fastly-only crate. It + /// also demonstrates that the *seam* is portable even though the crate is not. + struct SubstitutingAssembler; + + impl crate::platform::PlatformTemplateAssembler for SubstitutingAssembler { + fn assemble( + &self, + template: &str, + fragment: &str, + ) -> Result { + Ok(template.replace(ESI_BIDS_INCLUDE, fragment)) + } + } + fn services( http_client: Arc, cache: Arc, @@ -5770,6 +5826,7 @@ mod tests { .geo(Arc::new(NoopGeo)) .client_info(ClientInfo::default()) .template_cache(cache) + .template_assembler(Arc::new(SubstitutingAssembler)) .build() } @@ -5885,6 +5942,78 @@ mod tests { response.headers().get(name).and_then(|v| v.to_str().ok()) } + #[tokio::test] + async fn the_marker_never_reaches_the_browser_on_either_path() { + // The user-visible failure this whole chain exists to avoid: a page that + // returns 200, parses fine, renders no ads, and reports no error, because + // the `esi:include` was served literally. + // + // Both paths are checked. The miss path assembles after the transform; the + // hit path assembles after reading the cache. They are separate call sites + // and only one of them running would still look like success on the other. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + + let cold = String::from_utf8(body_of( + run(&settings, &services, navigation_request()).await, + )) + .expect("cold response should be utf-8"); + let warm = String::from_utf8(body_of( + run(&settings, &services, navigation_request()).await, + )) + .expect("warm response should be utf-8"); + + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the second request must be a hit, or this only tests one path" + ); + for (label, document) in [("miss", &cold), ("hit", &warm)] { + assert!( + !document.contains("esi:include"), + "{label}: an unresolved marker reached the browser: {document}" + ); + assert!( + document.contains("window.tsjs"), + "{label}: assembly must leave a bids script behind: {document}" + ); + } + } + + #[tokio::test] + async fn the_cached_template_holds_the_marker_and_never_the_bids() { + // The ordering the C3 prohibition depends on: store before assembling. If + // these were swapped, the cache would hold one visitor's bids and serve them + // to the next — and every test above would still pass, because the served + // page would look correct. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + + let entries = cache.entries.lock().expect("should lock entries"); + let stored = entries + .values() + .next() + .expect("a template should be stored"); + let template = core::str::from_utf8(&stored.body).expect("template should be utf-8"); + + assert!( + template.contains("esi:include"), + "the cached template must still carry the unresolved marker: {template}" + ); + assert!( + !template.contains("window.tsjs"), + "the cached template must not carry a bids script: {template}" + ); + } + #[tokio::test] async fn a_cache_hit_is_never_shared_cacheable() { // Asserted positively, because the obvious negative check does not work. From 4c557347a6a7ac368e96c2c76736b6fb90a5a83e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 13:56:42 +0530 Subject: [PATCH 201/395] Read the origin's Cache-Control, not the one TS just wrote MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local testing found that C2 never engaged on any page where the ad stack runs — which is every page that matters. Two origin fetches for two requests, and the esi:include served unresolved. TS stamps its own `private, no-store` on the response when should_run_ad_stack is true. The C2 gate ran after that stamp, read it as the origin's declaration, concluded OriginNotShareable, and refused. The gate asks whether the *origin* said the response was shareable, so it now runs before TS writes anything. The test suite could not have caught this, and that is the more important half of the fix. Its fixture left the auction disabled and passed no dispatch slots, so should_run_ad_stack was false in every test, the stamp never fired, and the ordering was unobservable. Every assertion about C2 was therefore made against the one configuration where C2's hardest condition does not apply. The fixture now runs the ad stack for real: auction enabled, plus a slot in AuctionDispatch rather than only in settings, since should_run_ad_stack requires a matched slot and the two are different inputs. sec-fetch-mode: navigate added for the same reason. Verified by mutation both ways. With the old fixture, moving the gate back below the stamp passed all seven tests. With the corrected fixture it fails six. The bug is now observable, which it was not before. Also verified end to end under viceroy serve against a stub origin: one origin fetch for two requests, no unresolved marker on either path, a bids script present in both, private, no-store on the hit, and the cached template 353 bytes against 467 served — so the cache holds the pre-assembly template. Full gates green — fmt, six clippy targets, four adapter suites. --- crates/trusted-server-core/src/publisher.rs | 125 ++++++++++++++------ 1 file changed, 89 insertions(+), 36 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8896522d5..ed2e59217 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3375,6 +3375,52 @@ pub async fn handle_publisher_request( // stored or validated as an origin representation. Strip both browser and // surrogate validators/cache directives before returning it. // + // The shared-template cache gate: it does not build the key, it authorizes storing + // the one built pre-fetch. Everything it checks is response-derived, which is + // exactly why it cannot run at lookup time — and why it does not need to. Anything + // already in the cache passed this gate on the way in. + // + // A surviving key is the store authorization. Under `Inline` there is no key to + // survive. + // + // **Evaluated before TS stamps its own `private, no-store` below, and that ordering + // is load-bearing.** The gate asks whether the *origin* declared the response + // shareable. Run it after the stamp and it reads TS's own header instead, concludes + // `OriginNotShareable`, and refuses to cache — on every page where the ad stack + // runs, which is every page that matters. Local testing caught exactly that; no unit + // test did, because their fixtures leave the auction disabled and never reach the + // stamp. + let gate_content_type = response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|h| h.to_str().ok()) + .unwrap_or_default() + .to_string(); + let template_cache_key = template_cache_key.filter(|_| { + match c2_bypass_reason( + assembly_mode, + request_had_authorization, + request_had_cookie, + response.status(), + &gate_content_type, + response.headers(), + &settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::template_cache_vary) + .unwrap_or_else(|| VarySpec::new([])), + ) { + Some(reason) => { + log::debug!("c2_template_cache bypass: {reason}"); + false + } + None => { + log::debug!("c2_template_cache eligible"); + true + } + } + }); + // Gate on `should_run_ad_stack` rather than content-type alone: when no slot // matched, the feature is disabled, or this is not an ad-eligible navigation, // no per-user `tsjs.adSlots`/`tsjs.bids` are injected, so forcing private @@ -3424,38 +3470,6 @@ pub async fn handle_publisher_request( .to_lowercase(); let route = classify_response_route(status, &content_type, &content_encoding, request_host); - // The shared-template cache gate: it does not build the key, it authorizes storing - // the one built pre-fetch. Everything it checks is response-derived, which is - // exactly why it cannot run at lookup time — and why it does not need to. Anything - // already in the cache passed this gate on the way in. - // - // A surviving key is the store authorization. Under `Inline` there is no key to - // survive. - let template_cache_key = template_cache_key.filter(|_| { - match c2_bypass_reason( - assembly_mode, - request_had_authorization, - request_had_cookie, - status, - &content_type, - response.headers(), - &settings - .creative_opportunities - .as_ref() - .map(CreativeOpportunitiesConfig::template_cache_vary) - .unwrap_or_else(|| VarySpec::new([])), - ) { - Some(reason) => { - log::debug!("c2_template_cache bypass: {reason}"); - false - } - None => { - log::debug!("c2_template_cache eligible"); - true - } - } - }); - match route { ResponseRoute::PassThrough => { log::debug!( @@ -5780,10 +5794,23 @@ mod tests { } } + /// Settings with the ad stack **live**, not merely configured. + /// + /// `[auction] enabled = true` and a slot matching the request path are both + /// required, because `should_run_ad_stack` folds them together and half this + /// path only executes when it is true. An earlier version of this helper left + /// the auction disabled, which made every test here exercise the branch where + /// TS never stamps its own `private, no-store` — and so missed that the C2 gate + /// was reading that stamp and refusing to cache every page that runs ads. fn settings_with_mode(mode: &str) -> Settings { let toml = format!( - "{}\n[creative_opportunities]\ngam_network_id = \"99999\"\n\ - assembly_mode = \"{mode}\"\n", + "{}\n[auction]\nenabled = true\n\n\ + [creative_opportunities]\ngam_network_id = \"99999\"\n\ + assembly_mode = \"{mode}\"\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"test-slot\"\n\ + page_patterns = [\"/article\"]\n\ + formats = [{{ width = 728, height = 90 }}]\n", crate_test_settings_str() ); let mut settings = @@ -5850,10 +5877,36 @@ mod tests { .uri("https://ts.example.com/article") .header(header::HOST, "ts.example.com") .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate") .body(EdgeBody::empty()) .expect("should build navigation request") } + /// A slot matching `/article`. + /// + /// Passed through `AuctionDispatch`, not read from settings — and that is the + /// distinction that matters. `should_run_ad_stack` requires a *matched* slot, so + /// a config slot with no dispatch slot leaves the ad stack off and skips every + /// branch that only runs when it is on. + fn article_slot() -> crate::creative_opportunities::CreativeOpportunitySlot { + crate::creative_opportunities::CreativeOpportunitySlot { + id: "test-slot".to_string(), + gam_unit_path: None, + div_id: Some("test-slot".to_string()), + page_patterns: vec!["/article".to_string()], + formats: vec![crate::creative_opportunities::CreativeOpportunityFormat { + width: 728, + height: 90, + media_type: MediaType::Banner, + }], + floor_price: None, + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + compiled_unit: None, + } + } + /// Runs the full request, **including the finalizer**. /// /// The finalizer is not optional here: the store happens once the transform has @@ -5879,7 +5932,7 @@ mod tests { &mut ec_context, AuctionDispatch { orchestrator: &orchestrator, - slots: &[], + slots: &[article_slot()], registry: None, }, request, @@ -6149,7 +6202,7 @@ mod tests { &mut ec_context, AuctionDispatch { orchestrator: &orchestrator, - slots: &[], + slots: &[article_slot()], registry: None, }, authenticated, From 76df2469194e6f20f3bef69c3af419553aeec5ff Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 14:08:31 +0530 Subject: [PATCH 202/395] Close the three Task 6 gates that do not need a deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-user leakage, stale revalidation, and transform failure. Each is a hard fail in the plan, independent of any performance result. Cross-user leakage is the one the design rests on. Two synthetic users differing in EC identity, consent jurisdiction and geo must store a byte-identical template — asserted as byte-identity rather than as a list of checks, because that does not depend on guessing which field might leak. Each user runs against a fresh cache, or the first user's entry would answer for the second and the comparison would prove nothing. The forbidden-substring assertions are the second layer: byte-identity would also hold if both templates leaked the same wrong thing. Transform failure covers a partial template reaching C2, which is the worst outcome available here — a truncated document served to every later visitor, indefinitely, with no error after the first request. Safe by construction, since the cap error propagates before the store; "by construction" is exactly the claim that stops holding after an unrelated refactor moves a line. The stale test needed rewriting because the first version passed for the wrong reason. A zero TTL produces an absent entry, not a stale one, so `is_stale()` was never reached — confirmed by reverting the staleness check and watching that version stay green. An entry is only present-and-stale with a stale_while_revalidate window, so the test now inserts one directly. With that fixed, the same mutation kills it. All three verified by mutation, which is the only reason to trust them: leaking adSlots through the head seam fails the leakage gate, storing before the cap check fails the transform gate, and serving stale fails the stale gate. Full gates green — fmt, six clippy targets, four adapter suites. --- .../src/template_cache.rs | 28 +++ crates/trusted-server-core/src/publisher.rs | 205 ++++++++++++++++++ 2 files changed, 233 insertions(+) diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs index 3a32a617e..8f472138e 100644 --- a/crates/trusted-server-adapter-fastly/src/template_cache.rs +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -269,6 +269,34 @@ mod tests { ); } + #[test] + fn a_stale_but_present_entry_reads_as_a_miss_rather_than_being_served() { + // Stale-while-revalidate is a real option and deliberately not taken: it is a + // state machine `cache::core` does not implement for you, and serving stale here + // means serving a template built by an older transform or an older JS bundle. + // + // The entry has to be *present and stale*, not merely expired. A zero TTL with no + // `stale_while_revalidate` window is simply absent, so a test written that way + // passes without ever reaching `is_stale()` — verified: reverting the staleness + // check left that version green. The revalidate window is what keeps the object + // readable while stale, so this actually exercises the branch. + let key = key("https://example.com/stale"); + let body = b"stale-template".to_vec(); + let metadata = metadata_for(&body); + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + let mut writer = fastly::cache::core::insert(cache_key, Duration::from_secs(0)) + .stale_while_revalidate(Duration::from_secs(60)) + .user_metadata(metadata.encode().into()) + .execute() + .expect("should begin insert"); + writer.write_all(&body).expect("should write body"); + writer.finish().expect("should finish insert"); + + let miss = run(cache().get(&key)).expect_err("a stale template must not be served"); + assert_eq!(miss, TemplateCacheMiss::NotFound); + } + #[test] fn purge_all_clears_stored_templates() { // The rollback lever. Without this, backing out a bad template means waiting diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index ed2e59217..c3768b53a 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -6067,6 +6067,70 @@ mod tests { ); } + #[tokio::test] + async fn a_transform_that_overruns_its_buffer_stores_nothing() { + // The 16 MB cap in production, shrunk here. A partial template in C2 is the + // worst outcome available: it would be served to every subsequent visitor as + // a truncated document, indefinitely, with no error after the first request. + // + // Safe by construction — the cap error propagates before the store runs — but + // "by construction" is exactly the kind of claim that stops being true after + // an unrelated refactor moves one line. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let mut raw = settings_with_mode("esi"); + raw.publisher.max_buffered_body_bytes = 8; + let settings = Arc::new(raw); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let publisher_response = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + navigation_request(), + ) + .await + .expect("the request itself should succeed; the cap trips during streaming"); + + let result = publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(&settings), + ®istry, + orchestrator, + services.clone(), + ) + .await; + + assert!( + result.is_err(), + "overrunning the buffer must fail rather than truncate" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "a failed transform must leave nothing in the shared cache" + ); + } + #[tokio::test] async fn a_cache_hit_is_never_shared_cacheable() { // Asserted positively, because the obvious negative check does not work. @@ -6133,6 +6197,147 @@ mod tests { ); } + /// Geo that reports a fixed, recognizable location. + /// + /// A distinct value per synthetic user, so a geo leak into the template shows up + /// as a substring rather than requiring inference. + struct StubGeo(&'static str); + + impl crate::platform::PlatformGeo for StubGeo { + fn lookup( + &self, + _client_ip: Option, + ) -> Result, Report> { + Ok(Some(GeoInfo { + city: self.0.to_string(), + country: self.0.to_string(), + continent: self.0.to_string(), + latitude: 1.0, + longitude: 2.0, + metro_code: 3, + region: Some(self.0.to_string()), + asn: Some(4), + })) + } + } + + /// One synthetic user: an identity, a consent posture, and a location. + struct SyntheticUser { + ec_id: &'static str, + jurisdiction: crate::consent::jurisdiction::Jurisdiction, + geo_marker: &'static str, + } + + /// Runs one synthetic user against a fresh cache and returns the stored template. + /// + /// Fresh cache per user deliberately: the point is to compare what each *would* + /// store, so sharing a cache would let the first user's entry answer for the + /// second and the comparison would prove nothing. + async fn stored_template_for(user: &SyntheticUser) -> Vec { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(Arc::clone(&stub) as Arc) + .geo(Arc::new(StubGeo(user.geo_marker))) + .client_info(ClientInfo::default()) + .template_cache( + Arc::clone(&cache) as Arc + ) + .template_assembler(Arc::new(SubstitutingAssembler)) + .build(); + queue_shareable_html(&stub); + + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let consent = crate::consent::ConsentContext { + jurisdiction: user.jurisdiction.clone(), + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(Some(user.ec_id.to_string()), consent); + let publisher_response = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + navigation_request(), + ) + .await + .expect("should proxy publisher request"); + let _ = publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(&settings), + ®istry, + orchestrator, + services.clone(), + ) + .await + .expect("should finalize publisher response"); + + let entries = cache.entries.lock().expect("should lock entries"); + entries + .values() + .next() + .expect("a template should have been stored") + .body + .clone() + } + + #[tokio::test] + async fn two_users_differing_in_identity_consent_and_geo_store_the_same_template() { + // The gate the whole design rests on. The template is shared between + // visitors, so anything request-scoped that reaches it is one visitor's data + // served to the next. Byte-identity is the assertion because it does not + // depend on guessing which field might leak. + let alice = SyntheticUser { + ec_id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.alice1", + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + geo_marker: "AliceCity", + }; + let bob = SyntheticUser { + ec_id: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.bobbb1", + jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, + geo_marker: "BobCity", + }; + + let alice_template = stored_template_for(&alice).await; + let bob_template = stored_template_for(&bob).await; + + assert_eq!( + alice_template, bob_template, + "two users differing in identity, consent and geo must produce the same \ + shared template" + ); + + // Belt and braces: byte-identity would also hold if *both* templates leaked + // the same wrong thing, so name the values that must be absent. + let template = String::from_utf8(alice_template).expect("template should be utf-8"); + for forbidden in [ + alice.ec_id, + bob.ec_id, + alice.geo_marker, + bob.geo_marker, + "adSlots", + "window.tsjs", + ] { + assert!( + !template.contains(forbidden), + "`{forbidden}` must not appear in a shared template: {template}" + ); + } + } + #[tokio::test] async fn inline_mode_never_reads_or_writes_the_cache() { // The shipped path. If this ever cached, per-user ad state would be shared From 00b8e0200a50a22930f7c1ebff7cbdcb898b0051 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 14:10:17 +0530 Subject: [PATCH 203/395] Record the local run, and the pattern the bugs on this branch share MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 6's local gates are marked done with what verified each. Two entries carry more than a checkbox. The C3 gate's wording is what caught a live bug, so the original wording is retained next to the result: checking for the absence of public/s-maxage/Surrogate-Control would have reported a hit serving with no Cache-Control at all as safe, because nothing was present to forbid. Request collapsing is left open rather than quietly dropped. Viceroy is single-threaded, so the concurrent cold-request case cannot be produced here; the racing-writer half is covered and the collapsing half is not. The findings document now records the local run and the bug it found — the C2 gate reading TS's own private, no-store as the origin's declaration, which disabled caching on every page that serves ads while every test passed. And the pattern across five bugs on this branch: each compiled, passed every existing test, and was wrong. Three were found by writing the test the plan asked for, one needed a running server, none by review — including my own review of the same gate, twice, in opposite directions. The stale-cache test is that failure in miniature: green while never reaching the branch it named, exposed only by mutation. Docs build verified, not just formatted. --- .../2026-08-08-1009-measurement-findings.md | 58 +++++++++++++++++++ .../2026-08-10-1009-esi-validation-spike.md | 42 +++++++++++--- 2 files changed, 92 insertions(+), 8 deletions(-) diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md index ea58c61b0..2ea4a7a48 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -383,6 +383,64 @@ themselves mutation-checked. **Still not deployable.** `ClientFill` and `Esi` render a template with a hole and nothing filling it — Task 4 and Task 5. A cache that works is necessary, not sufficient. +## Local end-to-end run — the Esi arm renders + +`viceroy serve` against a stub origin, config pushed into a scratchpad `fastly.toml` so +nothing tracked was modified. Served document: + +```html +

Stub article

+
+

Body copy.

+ +``` + +No `esi:include`. One origin fetch for two requests. `private, no-store` on the hit. +Cached template 353 bytes against 467 served, so the cache holds the pre-assembly +template. All three fragment formats behave: script, JSON, and `400` on a typo. `Inline` +unaffected — two fetches for two requests, no C2 activity, no markers. + +### The bug only a running server could find + +With the auction **enabled**, C2 never engaged: two origin fetches, marker unresolved. + +TS stamps its own `private, no-store` when `should_run_ad_stack` is true. The C2 gate ran +after that stamp, read it as the origin's declaration, concluded `OriginNotShareable`, and +refused — **on every page that serves ads**, which is every page that matters. + +The more important half is why no test caught it. The fixture left the auction disabled +and passed `slots: &[]`, so `should_run_ad_stack` was false in every test, the stamp never +fired, and the ordering was unobservable. Every C2 assertion had been made against the one +configuration where C2's hardest condition does not apply. + +Demonstrated both ways: with the old fixture, reintroducing the bug passes all seven +tests; with the corrected fixture it fails six. + +### Pattern across this branch + +Five bugs now share one shape — compiled, passed every existing test, and were wrong: + +1. The head-seam gate silently disabled body-close injection (`d9e05973`). +2. The key held the encoding the origin _chose_, so the cache could never hit (`2a2e6c6a`). +3. A C2 hit served with no `Cache-Control` at all (`0adb578e`). +4. A C2 hit dropped its in-flight auction, billing SSPs for nothing (`b3ac59a6`). +5. The gate read TS's own header as the origin's (`4c557347`). + +Three were found by writing the test the plan asked for. One needed a running server. None +were found by review — including my own, twice over on the same gate. + +The stale-cache test is the same failure in miniature: it passed while never reaching +`is_stale()`, and only mutation testing exposed that. A test that passes for the wrong +reason is worse than no test, because it is counted as coverage. + ## Step B — consumers of TS's own response headers Not yet run. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index a62006566..592646722 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -830,14 +830,32 @@ non-failed output — it is not first-success-wins. Not a phase. Every one of these is a hard fail, independent of any performance result. -- [ ] **Zero cross-user leakage.** Request the same URL as two synthetic users differing - in consent state, EC identity, and geo. Assert the C2 template is byte-identical - and that no bid, EC ID, consent string, or geo value appears in it. -- [ ] **Cold MISS, warm HIT, stale revalidation** each produce a correct page. -- [ ] **Transform failure** (the 16 MB buffer cap, a malformed body) does not insert a - partial template into C2 and does not serve one. +- [x] **Zero cross-user leakage.** DONE — `76df2469`. Two synthetic users differing in EC + identity, consent jurisdiction and geo store a byte-identical template, each against + a fresh cache so the first cannot answer for the second. Forbidden-substring checks + are the second layer, since byte-identity also holds if both leak the same thing. + Mutation-verified: leaking `adSlots` through the head seam fails it. +- [x] **Cold MISS, warm HIT, stale revalidation** DONE — `76df2469`, and end to end under + `viceroy serve` (below). Stale reads as a miss; serving stale would mean serving a + template built by an older transform or bundle. + + The first stale test passed for the wrong reason and had to be rewritten: a zero TTL + produces an *absent* entry, not a stale one, so `is_stale()` was never reached — + confirmed by reverting the check and watching it stay green. Only a + `stale_while_revalidate` window makes an entry present-and-stale. + +- [x] **Transform failure** DONE — `76df2469`. A partial template in C2 is the worst + outcome available: a truncated document served to every later visitor, indefinitely, + with no error after the first request. Mutation-verified by storing before the cap + check. - [ ] **Request collapsing** works: concurrent cold requests transform once. -- [ ] **DCA disabled**, verified by the injection test in Task 5 Step 2. +- [x] **DCA disabled** DONE — `0597f54e`. Config asserted _and_ behaviour: a fragment + carrying its own `esi:include` is spliced as text rather than dispatched. + +- [ ] **Request collapsing** — not tested, and not testable here. Viceroy is + single-threaded, so the concurrent cold-request case cannot be produced. The racing + _writer_ path is covered (`a_second_put_on_a_fresh_entry_is_a_no_op`), which is the + correctness half; the collapsing half needs real concurrency. - [ ] **Exactly one auction per pageview**, from `auction_events_raw`. - [ ] **Cookie and privacy finalization ran BEFORE assembly**, not after — EC `Set-Cookie` on first visit, geo suppression, and an unconditional @@ -849,7 +867,15 @@ Not a phase. Every one of these is a hard fail, independent of any performance r renders attributed. Use TS-attributed renders — the SSAT line item, non-empty `ts.bids`, `hb_adid` presence — **never slot fill**, which is blind to empty bids because `adInit` defines slots regardless. -- [ ] **No C3 — assert positively, not by absence.** Forbidding `public`, `s-maxage`, and +- [x] **No C3 — assert positively, not by absence.** DONE — `0adb578e`, and this gate's + wording caught a live bug. A C2 hit returns before the point where the publisher path + stamps `private, no-store`, so it served HTML with **no `Cache-Control` at all** — + heuristically cacheable, and therefore a shared cache of an assembled per-user + response. Checking for the _absence_ of `public`/`s-maxage`/`Surrogate-Control` would + have reported it as safe, because there was nothing present to forbid. Covered for + returning visitors specifically, where the cookie-privacy net never fires. + + Original wording, retained because it is what made the difference: Forbidding `public`, `s-maxage`, and `Surrogate-Control` is **not sufficient**: a bare `Cache-Control: max-age=60` passes that check and is still shared-cacheable, and that is exactly what the measured origin sends. Require instead that every assembled response carries From f1d9cb75f9afcbcb9597e0633039a994e0aef147 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 14:31:27 +0530 Subject: [PATCH 204/395] Architect streaming assembly, and correct the latency reading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The earlier finding said the hold costs approximately nothing. That is true today and only today, for a reason that stops holding the moment the rest of this work lands: the auction hides behind a slow origin fetch, so the hold costs max(0, auction - origin). Make the root cacheable, the origin fetch disappears, and the auction becomes the entire remaining cost. So the two problems are coupled and neither fix shows a win alone. The issue bundles them as one blocker; they need different fixes. Bids-in-the-body makes the page uncacheable and is done. Holding the response for the auction makes it slow and is not. The current implementation relocates the hold rather than removing it: a C2 hit awaits the auction, then assembles, then returns fully buffered. On a hit that is worse than today in one respect, because there is no origin fetch left to hide the auction behind. Three facts settle the design, each verified in the codebase rather than assumed. The existing streaming path already implements stream-then-stall-at-the-seam and is shipping. EdgeBody::Stream is an async stream, so an await may sit between chunks with no nested executor. BodyCloseInjection::Marker already exists and the streaming finalizers already strip Content-Length. Three designs compared. Buffered assembly is what exists. Native ESI via PendingFragmentContent::PendingRequest is what the crate is built for and is Fastly-only — and it vindicates the original dispatch gate, since under it the fragment request runs the auction and the root must not. Dispatch-usefulness turns out to be a function of the delivery mechanism, which is the non-obvious coupling here. The recommendation is neither: cache the shell with an inert HTML comment sentinel at the seam, split on it at serve time, stream the article, stall only for the auction, then write the bids and the tail. A comment rather than an esi:include because a comment is inert, so a substitution failure degrades to no ads instead of visible text in the page. Two simplifications fall out: store the template decoded and encode at serve time, which removes accept_encoding from the key entirely; and stop setting Content-Length, which is unknowable before bids resolve. The consequence for #1009 is the part worth reading. Its gating question is whether Fastly-first is acceptable for the flagship perf path. This design makes that question unnecessary — the full win on all four adapters, no esi dependency on the render path, no self-referencing backend, no second rendering architecture. ESI is sufficient but unnecessary: for one insertion point at a known location its parsing generality buys nothing a byte split does not. That is the opposite of what the issue expected. Docs build verified, not just formatted. --- ...11-1009-streaming-assembly-architecture.md | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md diff --git a/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md new file mode 100644 index 000000000..f911713c5 --- /dev/null +++ b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md @@ -0,0 +1,178 @@ +# Streaming assembly: the architecture #1009 actually needs + +**Date:** 2026-08-11 +**Status:** Decision record. Supersedes the delivery half of the +[ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md); the cache half +stands. +**Issue:** IABTechLab/trusted-server#1009 + +--- + +## 1. The correction this document exists for + +An earlier reading of the latency, recorded in +[the measurement findings](../plans/2026-08-08-1009-measurement-findings.md), said the +`` hold costs approximately nothing and the whole cost is the origin-cache bypass. + +That is true **today, and only today.** It is true for a reason that stops holding the +moment the rest of this work lands: + +| | Origin fetch | Auction | Reader waits | +| ----------------------------------- | ------------ | -------------------- | ------------ | +| Today | ~650 ms | hidden inside it | ~650–800 ms | +| Cached root, **buffered** assembly | 0 | fully exposed | ~auction cap | +| Cached root, **streaming** assembly | 0 | overlapped with send | ~ms | + +The auction is dispatched before the origin fetch and both run concurrently, so the hold +costs `max(0, auction − origin)` — zero while the origin is slow. Make the root cacheable +and the origin fetch disappears; the auction then has nothing left to hide behind and +becomes the _entire_ remaining cost. + +**So the two problems are coupled, and neither fix shows a win alone.** That is why the +issue is right to treat both as prerequisites, and why measuring one at a time misleads. + +Two distinct problems get bundled in the issue as one blocker. They need different fixes: + +1. **Bids live in the response body** → the page is _uncacheable_. Fixed by templatizing. + **Done.** +2. **The response is held for the auction** → the page is _slow_. Fixed by streaming the + shell and filling the seam late. **Not done** — this document. + +## 2. What the current implementation gets wrong + +On a C2 hit, `collect_and_assemble_cached_template` awaits the auction, then assembles, +then returns a fully buffered `PublisherResponse::Buffered`. The reader receives nothing +until bids resolve. + +That relocates the hold rather than removing it, and on a hit it is _worse than today_ in +one respect: there is no origin fetch left to hide it behind, so the full auction latency +lands on first byte. + +The routing decision that caused it — shared modes take the buffered finalizer — was made +because **a store needs complete transformed bytes.** True on a miss. Irrelevant on a hit, +where the template is already materialized. + +## 3. The decisive facts + +Three, all verified in the codebase rather than assumed: + +1. **The existing streaming path already implements stream-then-stall-at-the-seam.** + `publisher.rs` builds an `async_stream::try_stream!` that streams body chunks and holds + **only** at `` for the auction (`hold_auction`, `AuctionHoldState`). This is + shipping behaviour, not new work. +2. **`EdgeBody::Stream` is an async stream** — consumers call `stream.next().await` — so an + `await` may sit between chunks. Nothing needs a nested executor. +3. **`BodyCloseInjection::Marker(String)` already exists**, and the streaming finalizers + already strip `Content-Length`. + +## 4. Three designs + +| | Streams | Auctions | Requires | Adapters | +| ----------------------------------- | ------- | -------------- | ------------------------ | --------- | +| **A** — buffered assembly (current) | No | 1 | nothing | Fastly | +| **B** — native ESI subrequest | Yes | 1, in fragment | self-referencing backend | Fastly | +| **C** — cached shell + seam split | Yes | 1 | nothing | **All 4** | + +### Design B, for the record + +`PendingFragmentContent::PendingRequest` is what the `esi` crate is built for: the +dispatcher fires a real subrequest and the processor blocks on the handle. Fastly's +`send_async`/`wait` is **synchronous**, so this sidesteps the sync-dispatcher problem +without any executor. + +It also vindicates the _original_ dispatch gate. Under B the root must **not** dispatch, +because the fragment request runs the auction. The later reversal to +`root_auction_is_useful(Esi) = true` is correct for buffered assembly and wrong for +streaming. **Dispatch-usefulness is a function of the delivery mechanism**, which is the +non-obvious coupling in this design space. + +### Design C — the recommendation + +The template carries an **inert HTML comment sentinel** where the bids go, emitted by the +existing `Marker` variant: + +``` + +``` + +On a C2 hit: + +``` +commit headers (private, no-store; no Content-Length) ← must precede any byte on Fastly +stream template[..sentinel] ← the article paints here +await the auction ← the only stall, at the very end +write the bids script +stream template[sentinel+len..] +``` + +Since a hit has the whole template in hand, this is a `split_once`, not a streaming +search. Three yields from a `try_stream!`. + +**Why a comment sentinel rather than a byte offset in metadata.** An offset is O(1), but +capturing it means plumbing the writer position into a `lol_html` end-tag handler, and it +does not survive re-encoding. A `find` over a ~100 KB buffered template is free by +comparison. + +**Why a comment rather than `esi:include`.** An HTML comment is inert. If assembly ever +fails to substitute, the reader sees nothing; an unresolved `esi:include` renders as +visible text. Failure degrades to "no ads" instead of "broken page". + +**Why not re-run `lol_html` over the cached template.** It would inject a second tsjs +`content"#, + true, + ) .expect("should process HTML"); let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); @@ -1000,8 +1003,21 @@ mod tests { "should omit the DataDome client configuration" ); assert!( - !html.contains("/integrations/datadome/tags.js"), - "should omit the DataDome client tag URL" + html.contains("id=\"publisher-datadome\""), + "should preserve the publisher-originated DataDome tag" + ); + assert!( + html.contains("src=\"/integrations/datadome/tags.js\""), + "should rewrite the publisher-originated DataDome tag" + ); + assert!( + !html.contains("https://js.datadome.co/tags.js"), + "should remove the original third-party DataDome URL" + ); + assert_eq!( + html.matches("/integrations/datadome/tags.js").count(), + 1, + "should leave exactly one publisher-originated DataDome tag" ); } diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 2486c16be..75e6c3fdc 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -68,6 +68,7 @@ use serde_json::Value as JsonValue; use url::Url; use validator::Validate; +use crate::constants::ENV_FASTLY_IS_STAGING; use crate::error::TrustedServerError; use crate::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, @@ -469,6 +470,17 @@ impl DataDomeIntegration { Self::try_new(config).map(|_| ()) } + fn active_protection_test_bypass(&self) -> Option<&ProtectionTestBypassConfig> { + if std::env::var(ENV_FASTLY_IS_STAGING).as_deref() != Ok("1") { + return None; + } + + self.config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + } + fn validate_protection_test_bypass( config: &DataDomeConfig, ) -> Result<(), Report> { @@ -926,18 +938,26 @@ fn build( }; let integration = DataDomeIntegration::try_new(config)?; - let protection_test_bypass = integration + let protection_test_bypass_configured = integration .config .protection_test_bypass .as_ref() .is_some_and(|bypass| bypass.enabled); + let protection_test_bypass_active = integration.active_protection_test_bypass().is_some(); + if protection_test_bypass_configured && !protection_test_bypass_active { + log::warn!( + "[datadome] DataDome test bypass is configured but inactive because FASTLY_IS_STAGING is not 1" + ); + } log::info!( "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: {})", integration.config.sdk_origin, integration.config.rewrite_sdk, integration.config.enable_protection, - if protection_test_bypass { - "enabled" + if protection_test_bypass_active { + "active" + } else if protection_test_bypass_configured { + "configured-inactive" } else { "disabled" }, diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index bc06214ff..f2803b38f 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -165,15 +165,11 @@ impl DataDomeIntegration { req: &mut Request, services: &RuntimeServices, ) -> bool { - let Some(bypass) = self - .config - .protection_test_bypass - .as_ref() - .filter(|bypass| bypass.enabled) - else { + let value = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS); + let Some(bypass) = self.active_protection_test_bypass() else { return false; }; - let Some(value) = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS) else { + let Some(value) = value else { return false; }; @@ -806,6 +802,23 @@ mod tests { .expect("should build filter request") } + fn filter_with_staging( + integration: &DataDomeIntegration, + settings: &Settings, + services: &RuntimeServices, + request: &mut Request, + ) -> RequestFilterDecision { + temp_env::with_var(crate::constants::ENV_FASTLY_IS_STAGING, Some("1"), || { + futures::executor::block_on(integration.filter_protection_request(RequestFilterInput { + settings, + services, + request, + geo_info: None, + is_integration_route: false, + })) + }) + } + fn filter_marks_request( config: DataDomeConfig, services: &RuntimeServices, @@ -875,15 +888,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), ); - let decision = futures::executor::block_on(integration.filter_protection_request( - RequestFilterInput { - settings: &settings, - services: &services, - request: &mut request, - geo_info: None, - is_integration_route: false, - }, - )); + let decision = filter_with_staging(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -906,6 +911,148 @@ mod tests { ); } + #[test] + fn protection_test_bypass_header_is_stripped_when_unconfigured_or_disabled() { + for protection_test_bypass in [ + None, + Some(ProtectionTestBypassConfig { + enabled: false, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ] { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass, + ..DataDomeConfig::default() + }; + let integration = + DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("stale-test-credential"), + ); + + let decision = filter_with_staging(&integration, &settings, &services, &mut request); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an allowed Protection API response should continue" + ); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "the bypass header must be stripped when the bypass is unconfigured or disabled" + ); + assert!( + !has_client_tag_suppression_marker(&request), + "an inactive bypass must not suppress the DataDome client tag" + ); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "an inactive bypass must still call the Protection API" + ); + } + } + + #[test] + fn protection_test_bypass_is_inactive_outside_staging() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + ); + + let decision = temp_env::with_var( + crate::constants::ENV_FASTLY_IS_STAGING, + None::<&str>, + || { + futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )) + }, + ); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an allowed Protection API response should continue" + ); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "the bypass credential must be stripped outside staging" + ); + assert!( + !has_client_tag_suppression_marker(&request), + "the bypass must not suppress the DataDome client tag outside staging" + ); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "the bypass must still call the Protection API outside staging" + ); + } + #[test] fn protection_test_bypass_wins_over_other_exclusions() { let config = DataDomeConfig { @@ -944,15 +1091,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), ); - let decision = futures::executor::block_on(integration.filter_protection_request( - RequestFilterInput { - settings: &settings, - services: &services, - request: &mut request, - geo_info: None, - is_integration_route: false, - }, - )); + let decision = filter_with_staging(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -1007,15 +1146,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("wrong-credential"), ); - let decision = futures::executor::block_on(integration.filter_protection_request( - RequestFilterInput { - settings: &settings, - services: &services, - request: &mut request, - geo_info: None, - is_integration_route: false, - }, - )); + let decision = filter_with_staging(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 291a54242..16cbac868 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1,4 +1,4 @@ -use std::any::Any; +use std::any::{Any, TypeId}; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; @@ -98,17 +98,19 @@ pub struct IntegrationScriptContext<'a> { pub document_state: &'a IntegrationDocumentState, } +type IntegrationDocumentStateMap = BTreeMap<(&'static str, TypeId), Arc>; + /// Per-document state shared between HTML/script rewriters and post-processors. /// /// This exists to support multi-phase HTML processing without requiring a second HTML parse. #[derive(Clone, Default)] pub struct IntegrationDocumentState { - inner: Arc>>>, + inner: Arc>, } impl std::fmt::Debug for IntegrationDocumentState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let keys: Vec<&'static str> = { + let keys: Vec<(&'static str, TypeId)> = { let guard = self .inner .lock() @@ -136,7 +138,7 @@ impl IntegrationDocumentState { .inner .lock() .expect("should lock integration document state"); - let value = guard.get(integration_id)?; + let value = guard.get(&(integration_id, TypeId::of::()))?; let cloned: Arc = Arc::clone(value); cloned.downcast::().ok() } @@ -159,17 +161,15 @@ impl IntegrationDocumentState { .lock() .expect("should lock integration document state"); - if let Some(existing) = guard.get(integration_id) + let key = (integration_id, TypeId::of::()); + if let Some(existing) = guard.get(&key) && let Ok(downcast) = Arc::clone(existing).downcast::() { return downcast; } let value: Arc = Arc::new(init()); - guard.insert( - integration_id, - Arc::clone(&value) as Arc, - ); + guard.insert(key, Arc::clone(&value) as Arc); value } @@ -1405,6 +1405,37 @@ mod tests { } } + #[test] + fn document_state_keeps_multiple_types_for_one_integration() { + let state = IntegrationDocumentState::default(); + let number = state.get_or_insert_with("test", || 7_u32); + let label = state.get_or_insert_with("test", || "first".to_string()); + let repeated_number = state.get_or_insert_with("test", || 99_u32); + + assert!( + Arc::ptr_eq(&number, &repeated_number), + "repeated insertion should preserve the original typed state" + ); + assert_eq!( + *state.get::("test").expect("should retrieve number"), + 7, + "should retain numeric state" + ); + assert_eq!( + state + .get::("test") + .expect("should retrieve label") + .as_str(), + "first", + "should retain string state under the same integration ID" + ); + assert_eq!( + label.as_str(), + "first", + "should return inserted string state" + ); + } + #[test] fn default_html_post_processor_should_process_is_false() { let processor = NoopHtmlPostProcessor; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 57c4be3db..79ebf4436 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1423,6 +1423,14 @@ pub async fn publisher_response_into_streaming_response( } } +/// Removes request headers that can produce a bodyless or partial origin response. +fn strip_conditional_and_range_headers(req: &mut Request) { + req.headers_mut().remove(header::IF_NONE_MATCH); + req.headers_mut().remove(header::IF_MODIFIED_SINCE); + req.headers_mut().remove(header::RANGE); + req.headers_mut().remove(header::IF_RANGE); +} + /// Returns `true` when a buffered publisher response should carry a body and a /// recomputed `Content-Length`. /// @@ -1464,6 +1472,8 @@ fn apply_datadome_client_tag_cache_privacy( HeaderValue::from_static("private, max-age=0"), ); } + response.headers_mut().remove(header::ETAG); + response.headers_mut().remove(header::LAST_MODIFIED); for header_name in CDN_CACHE_HEADERS { response.headers_mut().remove(*header_name); } @@ -2881,11 +2891,15 @@ pub async fn handle_publisher_request( } ); - if should_run_ad_stack { - req.headers_mut().remove(header::IF_NONE_MATCH); - req.headers_mut().remove(header::IF_MODIFIED_SINCE); - req.headers_mut().remove(header::RANGE); - req.headers_mut().remove(header::IF_RANGE); + let suppress_datadome_client_side_tag = req + .extensions() + .get::() + .is_some(); + if should_run_ad_stack || suppress_datadome_client_side_tag { + // The origin content type is not known yet, so request hints cannot safely + // narrow this to HTML without allowing 304 or 206 responses to bypass a + // response mutation that becomes necessary after the fetch. + strip_conditional_and_range_headers(&mut req); } // Only advertise encodings the rewrite pipeline can decode and re-encode. @@ -2912,14 +2926,6 @@ pub async fn handle_publisher_request( // without streaming support may reject the flag outright rather than // silently buffering, which would fail every publisher fetch. let request_method = req.method().clone(); - let suppress_datadome_client_side_tag = req - .extensions() - .get::() - .is_some(); - if suppress_datadome_client_side_tag { - req.headers_mut().remove(header::IF_NONE_MATCH); - req.headers_mut().remove(header::IF_MODIFIED_SINCE); - } let mut platform_request = PlatformHttpRequest::new(req, backend_name); if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); @@ -5339,6 +5345,8 @@ mod tests { .header(header::HOST, "publisher.example") .header(header::IF_NONE_MATCH, "\"cached-page\"") .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-page\"") .body(EdgeBody::empty()) .expect("should build conditional request"); req.extensions_mut() @@ -5351,18 +5359,19 @@ mod tests { .into_iter() .next() .expect("should record one outbound request"); - assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_NONE_MATCH.as_str())), - "suppressed requests must not forward If-None-Match" - ); - assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_MODIFIED_SINCE.as_str())), - "suppressed requests must not forward If-Modified-Since" - ); + for header_name in [ + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::RANGE, + header::IF_RANGE, + ] { + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), + "suppressed requests must not forward {header_name}" + ); + } } #[tokio::test] @@ -5529,6 +5538,8 @@ mod tests { .header("fastly-surrogate-control", "max-age=600") .header("cloudflare-cdn-cache-control", "max-age=600") .header("cdn-cache-control", "max-age=600") + .header(header::ETAG, "\"origin-tag\"") + .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") .body(EdgeBody::empty()) .expect("should build cacheable HTML response"); @@ -5566,6 +5577,12 @@ mod tests { response.headers().get("cdn-cache-control").is_none(), "suppressed HTML should not retain CDN-Cache-Control" ); + for header_name in [header::ETAG, header::LAST_MODIFIED] { + assert!( + !response.headers().contains_key(&header_name), + "suppressed HTML should not retain {header_name}" + ); + } let mut no_store_response = Response::builder() .status(StatusCode::OK) diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 2205ae892..c2b1447e3 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -22,7 +22,6 @@ pub const CDN_CACHE_HEADERS: &[&str] = &[ "fastly-surrogate-control", "cdn-cache-control", "cloudflare-cdn-cache-control", - "cdn-cache-control", ]; /// Forces cookie-bearing responses to stay private to shared caches. diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 35eda5d3e..ff2eb6aaa 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -169,7 +169,7 @@ A request is protected when all of the following are true: 5. The client IP does not match `protection_excluded_ip_cidrs` or any Config Store-backed CIDR source. 6. The client ASN is not listed in `protection_excluded_asns`. 7. No `protection_exclusion_rules` match. -8. The request does not contain a matching enabled `protection_test_bypass` credential. +8. The request does not contain a matching enabled `protection_test_bypass` credential while `FASTLY_IS_STAGING=1`. Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, admin routes, discovery routes, and signature-verification routes are also excluded by default. @@ -182,6 +182,7 @@ can configure a static header credential that skips only the server-side Protection API: ```toml +# Runtime activation also requires FASTLY_IS_STAGING=1. [integrations.datadome.protection_test_bypass] enabled = true credential_secret_store = "ts_secrets" @@ -189,14 +190,17 @@ credential_secret_name = "datadome_test_bypass" ``` `protection_test_bypass` requires `enable_protection = true`; it is disabled -when omitted. Store the temporary credential in the configured Secret Store, -configure this section only while needed, protect the site with an outer access -control such as Basic Auth, and remove the section when testing finishes. Do not -enable it in production. - -The fixed `x-ts-datadome-bypass` header is compared in constant time, removed -before the request can reach DataDome or the publisher origin, and never -logged. Scope the header to the staging origin; do not attach it to every +when omitted and is runtime-active only when `FASTLY_IS_STAGING=1`. A retained +section cannot bypass protection in a production or other non-staging runtime. +Store the temporary credential in the configured Secret Store, configure this +section only while needed, protect the site with an outer access control such +as Basic Auth, and remove the section when testing finishes. + +Whenever the enabled DataDome request filter runs, the fixed +`x-ts-datadome-bypass` header is removed before configuration or credential +checks. It therefore cannot reach DataDome or the publisher origin when the +bypass is absent, disabled, inactive, or invalid. Active credentials are +compared in constant time and never logged. Scope the header to the staging origin; do not attach it to every request in a browser context because that can disclose the credential to third-party origins. With Playwright: @@ -223,7 +227,7 @@ This behavior applies to: - `protection_excluded_ip_cidr_sources`; - structured `ip_cidr` rules; - structured `ip_cidr_source` rules; and -- a matching enabled `protection_test_bypass` credential. +- a matching enabled `protection_test_bypass` credential in a staging runtime. ASN, method, path, query-parameter, static-asset, and internal-route exclusions do not automatically suppress the client-side tag. DataDome tags diff --git a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md index 0dfc923cf..2d9b0eae2 100644 --- a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md +++ b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md @@ -136,7 +136,6 @@ matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" - [ ] **Step 3: Make `filter_protection_request` own a mutable input and pass it mutably to `is_request_protected`.** In the existing `ProtectionScopeDecision::Skip` arm: - 1. determine whether the reason is IP-based; 2. if so, insert the typed marker into `input.request.extensions_mut()`; 3. call the updated skip logger with `client_tag_omitted = true`; and @@ -160,7 +159,6 @@ matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" optional Config Store data, and a mutable request. For each case, call `filter_protection_request`, assert it returns `Continue`, and inspect the request extension: - - inline `protection_excluded_ip_cidrs` match → marker present; - `protection_excluded_ip_cidr_sources` match → marker present; - structured `ProtectionMatcherConfig::IpCidr` match → marker present; @@ -220,7 +218,6 @@ DataDomeClientTagSuppressed request extension - [ ] **Step 2: Add a boolean to the owned and borrowed publisher-processing parameter structs.** Add a clearly named field such as `suppress_datadome_client_side_tag` to: - - `OwnedProcessResponseParams`; - `ProcessResponseParams`; and - `HtmlStreamProcessorParams`. @@ -241,7 +238,6 @@ DataDomeClientTagSuppressed request extension - [ ] **Step 4: Extend `IntegrationHtmlContext`.** Add the boolean as immutable request-scoped context. Populate it at both construction sites in `html_processor.rs`: - - the streaming `` element handler; and - `HtmlWithPostProcessing::process_chunk` for full-document post-processors. @@ -249,7 +245,6 @@ DataDomeClientTagSuppressed request extension `false` by default. - [ ] **Step 5: Add plumbing tests.** - - `HtmlProcessorConfig::from_settings` defaults to non-suppressed. - A test head injector records the context flag and sees `true` when a config is built with suppression. @@ -332,7 +327,6 @@ optimization. with a processable HTML content type, suppression `true`, and cacheable origin headers (`Cache-Control`, `Surrogate-Control`, and `Fastly-Surrogate-Control`). Assert the stream response is: - - `Cache-Control: private, max-age=0`; and - missing both surrogate cache headers. @@ -343,7 +337,6 @@ optimization. 204/205/304, or responses without suppression: none has a body variation created by this feature. - [ ] **Step 3: Add non-regression cache tests.** Verify that: - - non-suppressed processed HTML keeps its existing cache headers unless another existing policy changes them; - a suppressed CSS/non-HTML stream is not made private by this feature; and From 7828af12b3a04a3f9e7d115a88925711600faf45 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 19:11:17 +0530 Subject: [PATCH 212/395] Make the seam marker observable, since it is invisible by design MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repeatedly asked when the esi:include would appear in page source. It never will, in any mode: the marker is a hole in the *cached* copy, and assembly fills it before the response is sent on both the miss and hit paths. Seeing it in the browser was the bug fixed in the previous commit — it reached the page precisely because nothing resolved it. That left no way to confirm the mechanism works other than trusting that a store happened. The store log now reports whether the marker is present in the bytes going into the cache, and the harness shows that next to what the reader receives: The cached template (the shared copy — has a hole where bids go): c2_template_cache stored 738 bytes (seam marker present: true) What the reader receives (hole filled, no marker): 2 window.tsjs `seam marker present: true` is the evidence that the stored copy is genuinely reader-agnostic rather than carrying one reader's bids — which is the property the whole design depends on and the one a log line saying only "stored N bytes" cannot show. --- crates/trusted-server-core/src/publisher.rs | 15 ++++++++++++++- scripts/c2-local-test.sh | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 9580781b5..87073843b 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1530,7 +1530,20 @@ async fn store_template_if_authorized( .put(&key, &metadata, bytes.to_vec()) .await { - Ok(()) => log::debug!("c2_template_cache stored {} bytes", bytes.len()), + Ok(()) => { + // Reports whether the seam marker made it into the stored bytes. The marker + // is deliberately invisible from the outside — assembly replaces it before + // the response is sent, on both the miss and hit paths — so this log line is + // the only way to confirm the template really has a hole in it rather than + // per-reader bids baked in. + log::debug!( + "c2_template_cache stored {} bytes (seam marker present: {})", + bytes.len(), + bytes + .windows(ESI_BIDS_INCLUDE.len()) + .any(|w| w == ESI_BIDS_INCLUDE.as_bytes()) + ); + } Err(err) => log::warn!("c2_template_cache store failed: {err}"), } } diff --git a/scripts/c2-local-test.sh b/scripts/c2-local-test.sh index 4335c3ec2..8a2b2ccfe 100755 --- a/scripts/c2-local-test.sh +++ b/scripts/c2-local-test.sh @@ -244,6 +244,23 @@ else "$(( $(grep -c "POST /article" "$WORK/origin.log" || true) - POSTS_BEFORE ))" "1" fi +if [ "$MODE" != "inline" ]; then + info "Where the marker actually lives" + echo " The cached template (the shared copy — has a hole where bids go):" + grep -oE "c2_template_cache stored [0-9]+ bytes \(seam marker present: [a-z]+\)" \ + "$WORK/viceroy.log" | sort -u | sed 's/^/ /' + echo + echo " What the reader receives (hole filled, no marker):" + grep -oE "esi:include|window\.tsjs" "$WORK/r2.html" | sort | uniq -c | sed 's/^/ /' + cat <<'EOF' + + The marker is never visible in page source, in any mode. It exists only inside + the cache; assembly replaces it before the response is sent, on both the miss + and hit paths. `seam marker present: true` above is the evidence that the + stored copy is genuinely reader-agnostic rather than carrying someone's bids. +EOF +fi + info "Timing (bid endpoint delays $BID_DELAY s)" printf ' request 1 ttfb=%ss total=%ss\n' "$TTFB1" "$TOTAL1" printf ' request 2 ttfb=%ss total=%ss\n' "$TTFB2" "$TOTAL2" From 1761320fb8e62fc30748d4c05aee80435e75438f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 11 Aug 2026 19:52:00 +0530 Subject: [PATCH 213/395] Stream the cache hit, so the article arrives before the auction resolves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the half that turns a working cache into a latency win. A hit no longer assembles eagerly: the template's head goes out immediately, the auction is awaited at the seam, and the bids are written into the gap. Measured with a socket probe against the same stub, 1.5s bid endpoint: inline first body byte 15ms complete 1510ms esi hit, before first body byte 1524ms complete 1524ms esi hit, after first body byte 22ms complete 1511ms A new PublisherResponse::AssembleTemplate variant carries the template to the finalizer rather than assembling at the read. Two reasons: assembling eagerly is what held the first byte, and the finalizer owns the Arcs a 'static stream needs. It is distinct from Stream because the bytes are already transformed — running lol_html again would inject a second tsjs script and re-rewrite already-rewritten URLs. The compiler then forced every match to handle it, which is the point of a variant over a flag. Content-Length is now absent on a hit. The assembled length is unknown until bids resolve, and headers commit before the first body byte, so a length guessed here could not be corrected. A template that somehow has no seam marker is treated as a miss and refetched rather than served without ads. Two measurement corrections matter more than the code. curl's time_starttransfer reports the first byte of the *response*, which for a streaming response is the headers — committed long before any body byte. Every timing number I reported earlier was header-commit time. The direction held, because buffered assembly delayed headers too, but it could not have verified this fix: with the head yielded after the auction, curl still showed 24ms. The harness now uses a socket probe that finds the first byte past the header terminator, and that mutation shows 1511ms. And the unit test cannot cover this property at all. In-process there is no bid provider, so there is no auction to await and reordering the stream is unobservable — the mutation passes. The test asserts what it can (a hit streams, the first chunk is the document head and precedes the bids, no Content-Length); the timing assertion lives in the harness, where the delay is real. Both finalizers gained # Panics sections for the bid-state mutex. Full gates green, plus both harness modes: inline 5 passed, esi 8 passed. --- crates/trusted-server-core/src/publisher.rs | 356 ++++++++++++++++---- scripts/c2-local-test.sh | 85 ++++- 2 files changed, 361 insertions(+), 80 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 87073843b..23cfa02e9 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1146,6 +1146,27 @@ pub enum PublisherResponse { /// Parameters for [`process_response_streaming`]. params: Box, }, + /// A shared template read from C2, to be assembled on the way out. + /// + /// Distinct from [`Self::Stream`] because the bytes are **already transformed** — + /// running them through `lol_html` again would inject a second tsjs `\ + \ + \ + \ +

copy

{ESI_BIDS_INCLUDE}" + ); + + let assembled = assemble(&document, FRAGMENT).expect("should assemble"); + + for marker in ["", "", "", ""] { + assert!( + assembled.contains(marker), + "React Suspense marker {marker} must survive assembly, or hydration \ + fails: {assembled}" + ); + } + assert!( + assembled.contains("a\\u003eb & c"), + "inline script content must be byte-faithful: {assembled}" + ); + // Everything except the marker substitution must be untouched. + assert_eq!( + assembled, + document.replace(ESI_BIDS_INCLUDE, FRAGMENT), + "assembly must change nothing but the seam" + ); + } + #[test] fn script_bearing_fragments_are_spliced_verbatim() { // ESI substitutes bytes without escaping, which is exactly why the fragment diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 2ef4b8ec2..58a144ef1 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1356,6 +1356,25 @@ pub async fn buffer_publisher_response_async( ) .await?; let bytes = output.into_inner(); + // Decode before either storing or splicing. Both are textual operations and + // the transform's output follows the origin's encoding. + // + // Only shared modes pay this. Under `Inline` the bytes go out exactly as the + // encoder produced them, still compressed, which is what the encoder is for. + let bytes = if params.template_cache_key.is_some() { + let decoded = decode_transformed_body( + bytes, + ¶ms.content_encoding, + settings.publisher.max_buffered_body_bytes, + )?; + // The response now carries plaintext, so it must stop claiming otherwise. + response + .headers_mut() + .remove(http::header::CONTENT_ENCODING); + decoded + } else { + bytes + }; // Store first, assemble second — never the reverse. The stored bytes are // shared between visitors; the assembled ones carry this visitor's bids. // Swapping these two lines is the C3 leak. @@ -1421,6 +1440,38 @@ pub async fn buffer_publisher_response_async( } } +/// Decodes transformed body bytes so they can be stored and spliced as text. +/// +/// The pipeline pairs input encoding to output encoding, so a compressed origin yields a +/// compressed transform. Everything the shared-template path does afterwards is textual: +/// finding the seam marker, splitting on it, inserting a script. None of that works on +/// compressed bytes — the marker is not present to find, `from_utf8` fails, and a spliced +/// gzip stream is undecodable in the browser. +/// +/// An earlier attempt forced `Accept-Encoding: identity` on the *origin request* instead. +/// That worked and cost far too much: the origin then sent ~674 KB uncompressed where it +/// would have sent ~100 KB, adding seconds to the fetch. The fetch should stay +/// compressed; only the assembled response needs to be text. +/// +/// # Errors +/// +/// Returns an error if the bytes do not decode, which would mean the encoder and the +/// declared `Content-Encoding` disagree. +fn decode_transformed_body( + bytes: Vec, + content_encoding: &str, + max_decoded_bytes: usize, +) -> Result, Report> { + let compression = Compression::from_content_encoding(content_encoding); + if matches!(compression, Compression::None) { + return Ok(bytes); + } + let mut decoder = BodyStreamDecoder::new(compression, max_decoded_bytes); + let mut out = decoder.decode_chunk(bytes::Bytes::from(bytes))?.to_vec(); + out.extend_from_slice(&decoder.finish()?); + Ok(out) +} + /// Resolves the `` marker into this visitor's bids, if the mode assembles. /// /// Called *after* [`store_template_if_authorized`], never before: what is stored must @@ -1597,7 +1648,11 @@ async fn store_template_if_authorized( return; }; let metadata = crate::platform::TemplateMetadata { - content_encoding: params.content_encoding.clone(), + // `identity`, not the origin's encoding. The caller decoded before storing, + // because the seam split is textual — so recording the origin's encoding here + // would make a cache hit declare `Content-Encoding: gzip` over plaintext bytes, + // which is the same undecodable response one layer along. + content_encoding: "identity".to_string(), content_type: params.content_type.clone(), schema_version: crate::platform::TEMPLATE_SCHEMA_VERSION, body_len: bytes.len() as u64, @@ -3411,26 +3466,6 @@ pub async fn handle_publisher_request( // legacy path never sets it. Either way it is an internal edge signal that // must not leak to publisher backends. req.headers_mut().remove("fastly-ssl"); - // Shared modes ask the origin for identity, and this is not an optimization — - // without it the feature is broken end to end. - // - // The pipeline pairs input encoding to output encoding, so a gzip origin produces a - // gzip template. Every step after that assumes text: the seam marker cannot be found - // in compressed bytes, `String::from_utf8` on them fails outright, and splicing a - // plaintext bids script into the middle of a gzip stream yields - // `ERR_CONTENT_DECODING_FAILED` in the browser. Observed as a 502 on a real origin. - // - // The cost is real and accepted for the spike: a cache hit is served uncompressed, - // so it moves more bytes. Fixing that properly means storing decoded and re-encoding - // at serve time through a streaming encoder, which is a larger change than this - // spike needs to answer its question. - if !matches!(assembly_mode, AssemblyMode::Inline) { - req.headers_mut().insert( - header::ACCEPT_ENCODING, - HeaderValue::from_static("identity"), - ); - } - // The C2 key is built here, before the request is consumed, because every field // is request-derived and this is the last point where the request is in hand. // diff --git a/scripts/c2-local-test.sh b/scripts/c2-local-test.sh index 812cba1c1..bb79f957b 100755 --- a/scripts/c2-local-test.sh +++ b/scripts/c2-local-test.sh @@ -113,9 +113,11 @@ class H(BaseHTTPRequestHandler): # browser ERR_CONTENT_DECODING_FAILED. base = [("Cache-Control", "public, max-age=300"), ("Vary", "Accept-Encoding")] if "gzip" in (self.headers.get("Accept-Encoding") or ""): + print("origin: served COMPRESSED", flush=True) self._send(gzip.compress(PAGE), "text/html; charset=utf-8", base + [("Content-Encoding", "gzip")]) else: + print("origin: served PLAINTEXT", flush=True) self._send(PAGE, "text/html; charset=utf-8", base) def do_POST(self): @@ -241,6 +243,7 @@ else "$(grep -c 'window.tsjs' "$WORK/r2.html" || true)" "1" HDRS=$(curl -s -D- -o /dev/null -H "Host: ts.example.com" \ + -H "Accept-Encoding: gzip" \ -H "sec-fetch-dest: document" -H "sec-fetch-mode: navigate" \ "http://127.0.0.1:$TS_PORT/article") check "cache hit is not shared-cacheable" \ @@ -248,6 +251,7 @@ else POSTS_BEFORE=$(grep -c "POST /article" "$WORK/origin.log" || true) curl -s -o /dev/null -X POST -d 'x=1' -H "Host: ts.example.com" \ + -H "Accept-Encoding: gzip" \ "http://127.0.0.1:$TS_PORT/article" check "a POST still reaches the origin" \ "$(( $(grep -c "POST /article" "$WORK/origin.log" || true) - POSTS_BEFORE ))" "1" @@ -343,6 +347,12 @@ else # The property the unit tests cannot reach: in-process there is no bid provider, so # there is no auction to wait on and reordering the stream is unobservable. Here the # bid endpoint really sleeps, so the first body byte either beats it or does not. + # Guards a regression: an earlier fix forced Accept-Encoding: identity on the origin + # request, which made the origin send ~674KB where it would have sent ~100KB and added + # seconds to the fetch. Only the assembled response needs to be text; the fetch must + # stay compressed. + check "the origin fetch stays compressed" \ + "$(grep -c 'served PLAINTEXT' "$WORK/origin.log" || true)" "0" check "cache hit streams: the article is delivered before the auction resolves" \ "$(awk -v f="$FIRST_BODY" -v c="$COMPLETE" 'BEGIN { print (f < c / 3) ? "yes" : "no" }')" \ "yes" From a41d3e6eba474d5bc37b0faf3023b2fce1768dcb Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 12 Aug 2026 04:00:08 +0530 Subject: [PATCH 216/395] Assemble by byte seam, and carry slots through it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A review found the truncation diagnosis correct but "one change closes it" wrong. Six findings, all confirmed by reading the code rather than taken on trust. The worst was silent. Shared modes suppress the head slot script, and the seam carried only bids — so tsjs.adSlots stayed at its [] default, adInit defined nothing, and the page rendered perfectly while serving zero TS ads. My own comment supplied the alibi, claiming "the template already carries the slot markup", which conflated the publisher's div with TS's slot configuration. The harness asserted window.tsjs was present, not that adSlots was populated, so it passed green through exactly this. It now checks both, and refuses an empty array. Slots therefore travel on the seam with the bids, which is what the comment already claimed. That is also the only place they can go: they are request-gated, so baking them into a template shared between readers would decide for all of them. The remaining five: - Assembly no longer uses the esi crate. It loses content inside any element larger than its 16KB chunk_size, which is every Next.js RSC payload script. The byte split the hit path already used handles the real 1.4MB page intact. - Assembly is gated on the authorization, not the configured mode, and the authorization is read before the store consumes it. A bypassed response carries no marker, so the earlier draft turned an ordinary bypass — the common case — into a 500. - No slots means no seam at all rather than an empty one. Emitting an empty-slot seam still calls scheduleInitialAdInit, scheduling adInit for bots, prefetches and consent-denied requests: exactly the traffic that opted out. - Vary: * is refused. VarySpec::uncovered_by filters the wildcard out with a comment saying the eligibility gate handles it; nothing did, so a response the origin said no key can select was shareable. - Every assembled response is private, not only those where should_run_ad_stack is true. A suppressed request can still assemble an empty-bids document and would have kept the origin's public caching directives for a downstream cache to serve on. - Origin policy headers survive a hit. Reconstructing headers keeps Set-Cookie and caching directives out of a shared cache and also silently dropped Content-Security-Policy and framing protection. An allowlist stores the per-URL policy headers with the template, so anything per-reader or cache-controlling is excluded by construction. Marker validation is strict: missing or repeated is an error, since splicing the first of several would leave the rest in the page as visible text. The oversized-script test is inverted rather than deleted. It asserts the defect, because that defect is the reason the render path no longer uses the crate; if it starts failing, the crate has been fixed and the decision deserves revisiting. Full gates green, both harness modes pass (esi 11, inline 5). Known defect, deliberately not fixed here: current_bid_map recovers bids by un-escaping the rendered script and only reverses the two angle-bracket escapes, so a bid containing any other escaped character is lost. Every fixture has empty bids, so nothing catches it. The fix is to carry the bid map from write_bids_to_state rather than reconstruct it, which is a change in the auction collection path. --- .../src/esi_assembly.rs | 134 +++++ .../src/template_cache.rs | 1 + .../trusted-server-core/src/platform/mod.rs | 1 + .../src/platform/template_cache.rs | 47 +- crates/trusted-server-core/src/publisher.rs | 489 +++++++++++++++--- scripts/c2-local-test.sh | 7 + 6 files changed, 612 insertions(+), 67 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs index f7f6f87f0..70844268d 100644 --- a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs +++ b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs @@ -305,6 +305,140 @@ mod tests { ); } + #[test] + fn a_large_realistic_document_survives_byte_identically() { + // The real document is ~690KB of publisher HTML. Every other test here uses a + // handful of lines, and `Configuration` has a `chunk_size` — so a parser that is + // faithful on a small input can still corrupt one that spans many chunks. + // + // Observed on a real deployment: the cache-miss path, which runs this parser over + // the whole document, produced a broken page, while the cache-hit path, which + // does a plain byte split, produced a working one. This is that difference under + // test. + let mut document = String::from("t"); + for i in 0..6000 { + document.push_str(&format!( + "

copy {i} <tag>

\ +
" + )); + } + document.push_str(&format!("{ESI_BIDS_INCLUDE}")); + assert!( + document.len() > 600_000, + "fixture must be realistically large, got {}", + document.len() + ); + + let assembled = assemble(&document, FRAGMENT).expect("should assemble"); + + assert_eq!( + assembled.len(), + document.len() - ESI_BIDS_INCLUDE.len() + FRAGMENT.len(), + "assembled length must differ from the source by exactly the seam swap" + ); + assert_eq!( + assembled, + document.replace(ESI_BIDS_INCLUDE, FRAGMENT), + "a large document must survive byte-identically apart from the seam" + ); + } + + #[test] + fn a_document_larger_than_the_real_page_is_not_truncated() { + // The real page is ~1.4MB decoded. A deployment served 691704 bytes of it and + // the browser showed an error boundary — the document was cut roughly in half. + // The cache-hit path, which does a plain byte split, served the same page + // correctly, so the loss is in this parser and only shows up above some size the + // 600KB test does not reach. + let mut document = String::from("t"); + for i in 0..14000 { + document.push_str(&format!( + "

copy {i} <tag>

\ +
" + )); + } + document.push_str(&format!("{ESI_BIDS_INCLUDE}")); + assert!( + document.len() > 1_400_000, + "fixture must exceed the real page size, got {}", + document.len() + ); + + let assembled = assemble(&document, FRAGMENT).expect("should assemble"); + let expected = document.replace(ESI_BIDS_INCLUDE, FRAGMENT); + + assert_eq!( + assembled.len(), + expected.len(), + "assembly dropped {} bytes of a {}-byte document", + expected.len() as i64 - assembled.len() as i64, + document.len() + ); + assert!( + assembled.ends_with(""), + "the document must not be cut short; it ends with: {:?}", + &assembled[assembled.len().saturating_sub(80)..] + ); + } + + #[test] + fn dollar_signs_in_the_document_are_not_treated_as_esi_variables() { + // ESI interpolates `$(VAR)`, and a React Server Components payload is full of + // dollar signs: `[\"$\",\"$L1b\",null,…]` is how RSC encodes element references. + // + // A real page lost ~750KB through this parser while the byte-split path served it + // intact, and the two diverged exactly at `self.__next_f.push([1,"15:[\"$\",…`. + // Every fixture here until now was dollar-free. + let payload = r#""#; + let document = format!( + "

before

{payload}

after

{ESI_BIDS_INCLUDE}" + ); + + let assembled = assemble(&document, FRAGMENT).expect("should assemble"); + + assert!( + assembled.contains("after"), + "content after a dollar sign must survive: {assembled}" + ); + assert_eq!( + assembled, + document.replace(ESI_BIDS_INCLUDE, FRAGMENT), + "a document containing `$` must survive byte-identically apart from the seam" + ); + } + + #[test] + fn the_crate_truncates_a_script_larger_than_its_chunk_size() { + // Asserts the *defect*, deliberately. This is why the render path no longer uses + // this crate: `esi` 0.7.1 empties its buffer before parsing and never restores + // those bytes when the parser returns `Incomplete`, so any element larger than + // its 16KB `chunk_size` loses content. Next.js streams its RSC payload as a few + // enormous `self.__next_f.push(...)` scripts, so a real 1.4MB page came back at + // 697KB and the browser showed an error boundary. + // + // Total size is not the trigger — a 1.4MB document of small scripts is fine, and + // that is why every fixture here passed for weeks. The size of one element is. + // Raising `chunk_size` moves the threshold rather than removing it. + // + // If this ever starts failing, the crate has been fixed and edge assembly could + // be reconsidered on its merits rather than ruled out on this one. + let payload = "x".repeat(120_000); + let document = format!( + "

before

\ + \ +

after

{ESI_BIDS_INCLUDE}" + ); + + let assembled = assemble(&document, FRAGMENT).expect("should assemble"); + let expected = document.replace(ESI_BIDS_INCLUDE, FRAGMENT); + + assert!( + assembled.len() < expected.len(), + "the crate is expected to drop bytes here; if it no longer does, this test \ + and the decision it justifies both need revisiting" + ); + } + #[test] fn script_bearing_fragments_are_spliced_verbatim() { // ESI substitutes bytes without escaping, which is exactly why the fragment diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs index 8f472138e..d15f8ed55 100644 --- a/crates/trusted-server-adapter-fastly/src/template_cache.rs +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -193,6 +193,7 @@ mod tests { fn metadata_for(body: &[u8]) -> TemplateMetadata { TemplateMetadata { + policy_headers: Vec::new(), content_encoding: "identity".to_string(), content_type: "text/html; charset=utf-8".to_string(), schema_version: TEMPLATE_SCHEMA_VERSION, diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 8fae8a64a..b2e1c3446 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -57,6 +57,7 @@ pub use kv::UnavailableKvStore; pub use template_assembly::{ PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, }; +pub use template_cache::REPLAYABLE_POLICY_HEADERS; pub use template_cache::{ PlatformTemplateCache, TEMPLATE_SCHEMA_VERSION, TemplateCacheError, TemplateCacheKey, TemplateCacheMiss, TemplateEntry, TemplateMetadata, UnavailableTemplateCache, VarySpec, diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index 1b6c324ec..7057f49fd 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -143,6 +143,23 @@ fn surrogate_safe(url: &str) -> String { .collect() } +/// Origin response headers safe to store with a shared template and replay on a hit. +/// +/// Every one is a per-URL policy statement, identical for every reader. Nothing +/// per-reader (`Set-Cookie`) and nothing cache-controlling (`Cache-Control`, `ETag`, +/// `Surrogate-Control`) appears here, and it is an allowlist so a new origin header is +/// excluded until someone decides otherwise. +pub const REPLAYABLE_POLICY_HEADERS: &[&str] = &[ + "content-security-policy", + "content-security-policy-report-only", + "permissions-policy", + "referrer-policy", + "x-frame-options", + "x-content-type-options", + "content-language", + "x-robots-tag", +]; + /// Headers the key covers by construction, whatever the operator configured. /// /// `Accept-Encoding` has a dedicated key field ([`TemplateCacheKey::accept_encoding`]), @@ -265,6 +282,16 @@ pub struct TemplateMetadata { /// checking it on read makes a truncated entry a miss instead of a silently /// short template that would assemble into a broken page. pub body_len: u64, + /// Origin response headers that are policy, not per-reader state. + /// + /// Reconstructing headers from scratch on a hit keeps origin `Set-Cookie` and caching + /// directives out of a shared cache — but it also dropped `Content-Security-Policy`, + /// framing protection and `Content-Language`, weakening the page. These are + /// per-URL and identical for every reader, so they belong with the template. + /// + /// Deliberately an allowlist: anything per-reader or cache-controlling is excluded by + /// construction rather than by remembering to strip it. + pub policy_headers: Vec<(String, String)>, } impl TemplateMetadata { @@ -273,11 +300,16 @@ impl TemplateMetadata { /// unambiguous. #[must_use] pub fn encode(&self) -> Vec { - format!( + let mut out = format!( "v={}\nce={}\nct={}\nlen={}", self.schema_version, self.content_encoding, self.content_type, self.body_len - ) - .into_bytes() + ); + for (name, value) in &self.policy_headers { + // Header values cannot contain newlines (the HTTP parser rejects them), so a + // newline-delimited encoding cannot be broken by a header value. + out.push_str(&format!("\nh={name}:{value}")); + } + out.into_bytes() } /// Parse `user_metadata`. Returns `None` on anything unexpected, which callers @@ -286,6 +318,7 @@ impl TemplateMetadata { pub fn decode(raw: &[u8]) -> Option { let text = core::str::from_utf8(raw).ok()?; let mut schema_version = None; + let mut policy_headers = Vec::new(); let mut content_encoding = None; let mut content_type = None; let mut body_len = None; @@ -294,6 +327,11 @@ impl TemplateMetadata { match key { "v" => schema_version = Some(value.parse().ok()?), "ce" => content_encoding = Some(value.to_string()), + "h" => { + if let Some((name, header_value)) = value.split_once(':') { + policy_headers.push((name.to_string(), header_value.to_string())); + } + } "ct" => content_type = Some(value.to_string()), "len" => body_len = Some(value.parse().ok()?), _ => return None, @@ -301,6 +339,7 @@ impl TemplateMetadata { } Some(Self { schema_version: schema_version?, + policy_headers, content_encoding: content_encoding?, content_type: content_type?, body_len: body_len?, @@ -633,6 +672,7 @@ mod tests { fn metadata_round_trips() { let metadata = TemplateMetadata { content_encoding: "gzip".to_string(), + policy_headers: Vec::new(), content_type: "text/html; charset=utf-8".to_string(), schema_version: TEMPLATE_SCHEMA_VERSION, body_len: 42, @@ -674,6 +714,7 @@ mod tests { &key(), &TemplateMetadata { content_encoding: "identity".to_string(), + policy_headers: Vec::new(), content_type: "text/html".to_string(), schema_version: TEMPLATE_SCHEMA_VERSION, body_len: 0, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 58a144ef1..17be7dce6 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1244,6 +1244,12 @@ pub struct OwnedProcessResponseParams { /// /// Spike-only, for the #1009 ESI validation. pub(crate) template_cache_key: Option, + /// Slot definitions for the `` seam under a shared mode, as JSON. + /// + /// Request-scoped, so it travels with the request rather than into the template. + pub(crate) seam_ad_slots: Option, + /// Origin policy headers to store with the template and replay on a hit. + pub(crate) policy_headers: Vec<(String, String)>, pub(crate) content_encoding: String, pub(crate) origin_host: String, pub(crate) origin_url: String, @@ -1378,8 +1384,12 @@ pub async fn buffer_publisher_response_async( // Store first, assemble second — never the reverse. The stored bytes are // shared between visitors; the assembled ones carry this visitor's bids. // Swapping these two lines is the C3 leak. + // Read before the store: `store_template_if_authorized` *takes* the key so a + // request cannot store twice, which would leave nothing for assembly to gate + // on. + let was_authorized = params.template_cache_key.is_some(); store_template_if_authorized(services, &mut params, &bytes).await; - let bytes = assemble_if_shared(services, settings, ¶ms, bytes)?; + let bytes = assemble_if_shared(was_authorized, settings, ¶ms, bytes)?; response.headers_mut().insert( http::header::CONTENT_LENGTH, http::HeaderValue::from(bytes.len() as u64), @@ -1415,16 +1425,18 @@ pub async fn buffer_publisher_response_async( ) .await; } - let (head, tail) = split_template_at_seam(&template); - let bids = params - .ad_bids_state - .lock() - .expect("should lock bid state") - .clone() - .unwrap_or_else(build_empty_bids_script); - let mut assembled = Vec::with_capacity(head.len() + bids.len() + tail.len()); + let (head, tail) = split_template_at_seam(&template).change_context_lazy(|| { + crate::error::TrustedServerError::Proxy { + message: "cached template has no usable seam marker".to_string(), + } + })?; + let seam = build_seam_script( + params.seam_ad_slots.as_deref().unwrap_or("[]"), + ¤t_bid_map(¶ms.ad_bids_state), + ); + let mut assembled = Vec::with_capacity(head.len() + seam.len() + tail.len()); assembled.extend_from_slice(head); - assembled.extend_from_slice(bids.as_bytes()); + assembled.extend_from_slice(seam.as_bytes()); assembled.extend_from_slice(tail); response.headers_mut().insert( http::header::CONTENT_LENGTH, @@ -1472,55 +1484,91 @@ fn decode_transformed_body( Ok(out) } -/// Resolves the `` marker into this visitor's bids, if the mode assembles. +/// Splices this visitor's slots and bids into the seam, if the mode assembles. +/// +/// Uses the same byte split as the hit path, and deliberately **not** the `esi` crate. +/// That crate loses content inside any element larger than its 16 KB `chunk_size`: it +/// empties its buffer before parsing and never restores those bytes when the parser +/// returns `Incomplete`. Next.js streams its RSC payload as a few enormous +/// `self.__next_f.push(...)` scripts, so a real 1.4 MB page came back at 697 KB and the +/// browser showed an error boundary. Raising `chunk_size` moves the threshold rather +/// than removing it. /// -/// Called *after* [`store_template_if_authorized`], never before: what is stored must -/// be the template every visitor shares, and what is returned must be this visitor's -/// document. Two call sites rather than one so that ordering is visible rather than -/// implied. +/// Called *after* [`store_template_if_authorized`], never before: what is stored must be +/// the template every visitor shares. /// /// # Errors /// -/// Returns an error if the adapter has no assembler or if assembly fails. Deliberately -/// fatal rather than falling back to the unassembled template: that template contains a -/// literal `esi:include`, so serving it would render no ads, report no error, and look -/// to every monitor like a page that worked. +/// Returns an error if the seam marker is missing or repeated. fn assemble_if_shared( - services: &RuntimeServices, + was_authorized: bool, settings: &Settings, params: &OwnedProcessResponseParams, bytes: Vec, ) -> Result, Report> { - let assembly_mode = settings + // Gated on the *authorization*, not on the configured mode. A bypassed response fell + // back to inline and therefore carries no marker; splitting it would fail and turn + // an ordinary bypass — the common case against a real origin — into a 500. + // Both conditions, and neither alone: only `Esi` emits a marker, and only an + // authorized response has one to find. `ClientFill` is authorized but marker-free. + let emits_a_marker = settings .creative_opportunities .as_ref() .map(CreativeOpportunitiesConfig::assembly_mode) - .unwrap_or_default(); - if !matches!(assembly_mode, AssemblyMode::Esi) { + .is_some_and(|mode| matches!(mode, AssemblyMode::Esi)); + if !was_authorized || !emits_a_marker { return Ok(bytes); } - let template = String::from_utf8(bytes).change_context(TrustedServerError::Proxy { - message: "shared template is not valid UTF-8, so it cannot be assembled".to_string(), + let (head, tail) = split_template_at_seam(&bytes).change_context_lazy(|| { + crate::error::TrustedServerError::Proxy { + message: "shared template has no usable seam marker".to_string(), + } })?; + // `None` means the ad stack did not run — bot, prefetch, consent-denied, kill switch. + // Emitting an empty-slot seam would still call `scheduleInitialAdInit`, scheduling + // `adInit` for exactly the traffic that opted out. Emit nothing instead. + let seam = params + .seam_ad_slots + .as_deref() + .map(|slots| build_seam_script(slots, ¤t_bid_map(¶ms.ad_bids_state))) + .unwrap_or_default(); - // The auction already in flight is the fragment. `body_close_injection` emitted a - // constant marker into the template precisely so this substitution — not a - // subrequest — is what fills it. - let fragment = params - .ad_bids_state - .lock() - .expect("should lock bid state") - .clone() - .unwrap_or_else(build_empty_bids_script); - - services - .template_assembler() - .assemble(&template, &fragment) - .map(String::into_bytes) - .change_context(TrustedServerError::Proxy { - message: "failed to assemble the shared template".to_string(), - }) + let mut out = Vec::with_capacity(head.len() + seam.len() + tail.len()); + out.extend_from_slice(head); + out.extend_from_slice(seam.as_bytes()); + out.extend_from_slice(tail); + Ok(out) +} + +/// The bids for this request, recovered from the rendered bids script. +/// +/// # Known defect +/// +/// `ad_bids_state` holds a *JS-escaped* `", + html_escape_for_script(slots_json), + html_escape_for_script(&bids) + ) +} + +/// The slot definitions a shared-mode seam must carry, as JSON. +/// +/// Mirrors [`template_ad_slots_script`]'s gating: same `should_run_ad_stack` condition, +/// same slot set. The difference is only *where* it is delivered — the seam, per +/// request, rather than the head, into a shared template. +pub(crate) fn seam_ad_slots_json( + mode: AssemblyMode, + should_run_ad_stack: bool, + settings: &Settings, + matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + request_path: &str, +) -> Option { + if matches!(mode, AssemblyMode::Inline) || !should_run_ad_stack { + return None; + } + let co_config = settings.creative_opportunities.as_ref()?; + let section = co_config.section_for_path(request_path); + let slots: Vec = matched_slots + .iter() + .filter_map(|slot| build_slot_json(slot, co_config, §ion)) + .collect(); + Some( + serde_json::to_string(&slots) + .expect("serde_json::to_string of Vec should be infallible"), + ) +} + /// Build the empty-bids `"#; let params = OwnedProcessResponseParams { template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), diff --git a/scripts/c2-local-test.sh b/scripts/c2-local-test.sh index bb79f957b..e50ea9e75 100755 --- a/scripts/c2-local-test.sh +++ b/scripts/c2-local-test.sh @@ -241,6 +241,13 @@ else "$(grep -c 'esi:include' "$WORK/r2.html" || true)" "0" check "a bids script is present" \ "$(grep -c 'window.tsjs' "$WORK/r2.html" || true)" "1" + # `window.tsjs` alone passes while initial ads are dead: shared modes suppress the head + # slot script, so if the seam does not carry slots, `adSlots` stays `[]` and `adInit` + # defines nothing. This harness passed green through exactly that bug. + check "the seam carries slot definitions, not just bids" \ + "$(grep -c 'adSlots=JSON.parse' "$WORK/r2.html" || true)" "1" + check "the slot definitions are populated, not an empty array" \ + "$(grep -c 'adSlots=JSON.parse("\[\]")' "$WORK/r2.html" || true)" "0" HDRS=$(curl -s -D- -o /dev/null -H "Host: ts.example.com" \ -H "Accept-Encoding: gzip" \ From 2bc4624cd2c35ab16292226f9b1d39fb4bf5e74a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 12 Aug 2026 12:22:06 +0530 Subject: [PATCH 217/395] Make the shared-template path deliver bids, and make its tests notice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent reviews found eight defects on top of a41d3e6e. Every one had shipped past a fully green suite, because a fixture never reached the branch under test. The worst: ESI mode served zero bids. current_bid_map recovered them by un-escaping the rendered script, but html_escape_for_script escapes quotes, so any non-empty map was invalid JSON, from_str failed, and unwrap_or_default swallowed it into {}. Only the empty map survived — which is every fixture there was. AdBidsState now holds the rendered script and the structured map, both derived from one map in one call, so they cannot disagree. The rest: - Assembly is a byte split, never the esi crate. The crate loses content inside any element larger than its 16KB chunk_size, and Next.js streams its RSC payload as a few enormous scripts — a real 1.4MB page came back at 697KB. - Both hit finalizers emitted an empty-slot seam when the ad stack had not run, which still calls scheduleInitialAdInit and schedules adInit for bots, prefetches and consent-denied readers. Absent is not empty; all three call sites now share one helper. - The seam no longer assigns adSlots ahead of the navigation-generation guard, where a committed SPA route could clobber it. - The marker is an inert HTML comment, validated as exactly-one before a template is stored and before any response header commits, with TEMPLATE_SCHEMA_VERSION bumped so entries holding the old marker are never read back. - Cache-Control is read across all header lines, so a private on a second line disqualifies. The cache key's fingerprint folds in the integrations config, so reconfiguring one no longer reuses a template built under the old config. - client_fill's cache had never hit: the seam check was unconditional and client_fill emits no marker, so every hit failed as Missing and refetched. The requirement is now mode-aware via an exhaustive match, so a new mode must state its answer. Coverage was the actual failure. Two gaps are closed because reverting the fix left the suite green: the fingerprint was tested as a function but never through the call site into a real cache key, and client_fill's test used a fresh cache per reader with one request each — proving the stored bytes neutral, never that a second request is served from cache. Both now assert on what the cache actually did. The harness was lying too. It grepped gzipped bytes, so every content assertion passed for free; it counted log lines Viceroy emits twice; and it had no client_fill mode at all. It now gunzips before asserting, compares distinct values, runs all three modes, and checks a real winning bid reaches the page rather than accepting an empty auction. Verified by mutation throughout: each fix reverted, the test watched to fail, the fix restored. Full gates green — fmt, six clippy targets, four adapter suites, parity, 569 JS tests. Harness: inline 6/6, client_fill 12/12, esi 14/14. Two caveats recorded rather than fixed. client_fill now caches correctly but its client-side bid fetch lives only in the SPA navigation hook, so it likely still renders no ads on initial load. And the JS bundle is not content-hashed and is served max-age=300, so for a few minutes after a deploy an old one-argument scheduler can meet a new two-argument seam and silently drop slots; the schema version protects the document side, nothing versions the client side. assembly_mode still defaults to inline, and inline behaviour is unchanged. --- .../src/esi_assembly.rs | 18 +- .../trusted-server-core/src/html_processor.rs | 2 +- .../src/integrations/gpt_bootstrap.js | 6 +- .../src/platform/template_cache.rs | 7 +- crates/trusted-server-core/src/publisher.rs | 1644 +++++++++++++++-- .../trusted-server-js/lib/src/core/types.ts | 12 +- .../lib/src/integrations/gpt/index.ts | 16 +- .../integrations/gpt/gpt_bootstrap.test.ts | 30 + .../gpt/schedule_initial_ad_init.test.ts | 63 + scripts/c2-local-test.sh | 162 +- 10 files changed, 1723 insertions(+), 237 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs index 70844268d..5959904ba 100644 --- a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs +++ b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs @@ -140,7 +140,18 @@ impl PlatformTemplateAssembler for FastlyTemplateAssembler { #[cfg(test)] mod tests { use super::*; - use trusted_server_core::publisher::ESI_BIDS_INCLUDE; + + /// A hand-written include, no longer the seam's marker. + /// + /// It used to be `trusted_server_core::publisher::ESI_BIDS_INCLUDE`, on the + /// reasoning that a test writing its own marker would keep passing after the + /// seam's shape changed. That reasoning expired with the seam: the marker is now + /// an inert HTML comment (`SEAM_BIDS_MARKER`), which this crate cannot resolve and + /// is not meant to. What is left under test here is the `esi` crate's own + /// behaviour over an ESI document — including + /// [`the_crate_truncates_a_script_larger_than_its_chunk_size`], the defect that + /// took the crate out of the render path in the first place. + const ESI_BIDS_INCLUDE: &str = ""; const FRAGMENT: &str = ""; @@ -150,9 +161,8 @@ mod tests { #[test] fn the_seams_own_marker_is_resolved() { - // Deliberately built from `ESI_BIDS_INCLUDE` rather than a hand-written - // include. The two live in different crates, and a test that wrote its own - // marker would keep passing after the seam's changed shape stopped parsing. + // The processor must resolve a well-formed include. This is the crate's + // contract, not the seam's — the seam no longer emits ESI at all. let assembled = assemble(&template_with_include(), FRAGMENT).expect("should assemble"); assert!( diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index f0d21a7ad..d10c0252f 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -173,7 +173,7 @@ pub enum BodyCloseInjection { /// Read the auction result from `ad_bids_state` and inject it, falling back to /// an empty payload. Today's shipped behaviour. InlineBids, - /// Emit this markup verbatim — an `` for the edge to assemble. + /// Emit this markup verbatim — an inert marker the assembly step splits on. /// Must be identical for every request that reaches the transform, or the /// cached template is not shared-safe. Marker(String), diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index a6408b693..55607296d 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -94,8 +94,12 @@ // and deliberately identical to the bundle scheduler — the impression is // spent on a viewed tab, and the post-hydration guarantee holds whenever // the request is actually issued. - ts.scheduleInitialAdInit = function (initialBids) { + ts.scheduleInitialAdInit = function (initialBids, initialSlots) { if ((ts.navGeneration || 0) !== 0) return; + // Slots are generation-guarded for the same reason the bids are: the + // shared-template seam sends both, and an assignment made before this call + // would overwrite a committed SPA navigation's slots. + if (initialSlots) ts.adSlots = initialSlots; if (initialBids) ts.bids = initialBids; var fire = function () { if ((ts.navGeneration || 0) !== 0) return; diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index 7057f49fd..03dfa8b78 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -26,7 +26,12 @@ use crate::creative_opportunities::AssemblyMode; /// Bump on **any** change to what the transform emits. Without it a deploy reads /// yesterday's template shape and assembles against markers that moved, which fails /// as a rendering bug far from its cause rather than as a cache miss. -pub const TEMPLATE_SCHEMA_VERSION: u32 = 1; +/// +/// | Version | Transform | +/// | ------- | --------- | +/// | 1 | `` seam marker was `` | +/// | 2 | Marker is the inert comment [`SEAM_BIDS_MARKER`](crate::publisher::SEAM_BIDS_MARKER); the seam hands slots to `scheduleInitialAdInit` instead of assigning them | +pub const TEMPLATE_SCHEMA_VERSION: u32 = 2; /// Inputs that select one cached template. /// diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 17be7dce6..6ae816fdc 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -388,7 +388,7 @@ impl PublisherBodyProcessor { settings, integration_registry, ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), - ad_bids_state: Arc::clone(¶ms.ad_bids_state), + ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), gpt_diagnostics: params.gpt_diagnostics.clone(), shared_template_authorized: params.template_cache_key.is_some(), })?) @@ -1010,11 +1010,55 @@ pub(crate) fn root_auction_is_useful(mode: AssemblyMode) -> bool { } } -/// The `esi:include` emitted at the `` seam under [`AssemblyMode::Esi`]. +/// The marker emitted at the `` seam under [`AssemblyMode::Esi`], reserving the +/// place this reader's slots and bids are spliced into. +/// +/// An inert HTML comment, deliberately. It used to be an ``, from when the +/// `esi` crate resolved it at the edge — but that crate was removed from the render path +/// because it truncates any element larger than its 16 KB chunk size, and nothing has +/// parsed ESI since. What remained was a tag that *looked* executable, would have been +/// executed by any ESI-enabled layer in front of us, and renders as text in a browser if +/// assembly is ever skipped. A comment cannot do any of those things: an unassembled +/// template degrades to a page with no ads rather than a page with a visible tag. +/// +/// Carries no URL. Every byte here is a byte every reader of the shared template +/// receives, so nothing request-scoped may appear, and keeping a URL out also removes +/// any escaping question at the seam. +pub const SEAM_BIDS_MARKER: &str = ""; + +/// The mode the operator asked for, before availability is taken into account. +/// +/// Spelled once, because the mode has to mean the same thing at the cache key, at the +/// seam, and at both hit finalizers. Every one of those re-derived it from the same +/// `Option` chain, and the finalizers had no way to ask at all — which is why they +/// demanded a seam marker of a mode that emits none. +fn configured_assembly_mode(settings: &Settings) -> AssemblyMode { + settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::assembly_mode) + .unwrap_or_default() +} + +/// Whether this mode's `` seam emits [`SEAM_BIDS_MARKER`]. +/// +/// The property that decides whether a template is *expected* to have a hole in it, and +/// therefore whether the absence of one is a defect or the design. Only `Esi` splices per +/// reader; `ClientFill` deliberately injects nothing there, because the browser fetches +/// its own bids after load. +/// +/// Treating the marker as universal is what made `ClientFill`'s cache inert: every hit +/// failed the marker check, was logged as transform drift, and fell back to the origin, +/// so that mode stored templates it could never read back. /// -/// No `path` query parameter: see [`body_close_injection`]. The adapter's include -/// dispatcher appends it from the live request. -pub const ESI_BIDS_INCLUDE: &str = ""; +/// Matched exhaustively rather than compared against `Esi`, so a new mode has to state +/// its answer here instead of silently inheriting one. +fn mode_emits_seam_marker(mode: AssemblyMode) -> bool { + match mode { + AssemblyMode::Inline | AssemblyMode::ClientFill => false, + AssemblyMode::Esi => true, + } +} /// The assembly mode this response will actually be delivered under. /// @@ -1033,11 +1077,7 @@ pub const ESI_BIDS_INCLUDE: &str = " AssemblyMode { - let configured = settings - .creative_opportunities - .as_ref() - .map(CreativeOpportunitiesConfig::assembly_mode) - .unwrap_or_default(); + let configured = configured_assembly_mode(settings); if matches!(configured, AssemblyMode::Inline) || shared_template_authorized { return configured; } @@ -1055,21 +1095,9 @@ fn effective_assembly_mode(settings: &Settings, shared_template_authorized: bool /// two independent decisions: once [`template_ad_slots_script`] stopped emitting a /// head script under a shared mode, body-close injection stopped with it. /// -/// `Esi` emits an `esi:include` pointing at the page-bids endpoint's **fragment** -/// format, which returns the same executable `", html_escape_for_script(slots_json), html_escape_for_script(&bids) @@ -4641,15 +4867,22 @@ pub(crate) fn c2_bypass_reason( if !uncovered.is_empty() { return Some(C2BypassReason::VaryNotCovered(VaryGap(uncovered))); } - // One pass over the header. `private` and `no-store` match the cookie-privacy - // net's reading; `no-cache` is added because it means "revalidate before reuse" - // rather than "do not store" — correct for an HTTP cache, too permissive for a - // spike-owned one. + // Every value, not the first. `Cache-Control` may arrive as repeated header lines — + // `Cache-Control: public, max-age=300` then `Cache-Control: private` — and HTTP + // treats that identically to one comma-joined line. `HeaderMap::get` returns only + // the first, so a `private` or `no-store` in a later line was invisible and the + // response was stored in a cache shared between readers. Same fail-open shape as the + // `Vary` reads above, which is why those already use `get_all`. + // + // `private` and `no-store` match the cookie-privacy net's reading; `no-cache` is + // added because it means "revalidate before reuse" rather than "do not store" — + // correct for an HTTP cache, too permissive for a spike-owned one. let non_shareable = response_headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()) + .get_all(header::CACHE_CONTROL) + .iter() + .filter_map(|value| value.to_str().ok()) .map(str::to_ascii_lowercase) - .is_some_and(|value| { + .any(|value| { value.contains("private") || value.contains("no-store") || value.contains("no-cache") }); if non_shareable { @@ -5330,9 +5563,10 @@ mod tests { total_time_ms: 665, metadata: std::collections::HashMap::new(), }; - let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); + let state = AdBidsState::with_script("BIDS_SCRIPT"); prepend_auction_debug_comment("stream", &result, &state); let comment = state + .script_cell() .lock() .expect("should lock state") .clone() @@ -5391,9 +5625,10 @@ mod tests { total_time_ms: 12, metadata: std::collections::HashMap::new(), }; - let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); + let state = AdBidsState::with_script("BIDS_SCRIPT"); prepend_auction_debug_comment("stream", &result, &state); let comment = state + .script_cell() .lock() .expect("should lock state") .clone() @@ -5569,7 +5804,7 @@ mod tests { request_scheme: "https".to_owned(), content_type: "application/json".to_owned(), ad_slots_script: None, - ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -6012,6 +6247,145 @@ mod tests { } } + mod integration_fingerprint_tests { + use super::*; + + /// Base settings with one integration's config replaced. + /// + /// Edits the parsed `[integrations]` map rather than appending TOML, so the two + /// fixtures differ in exactly the field under test — the base settings already + /// declare `[integrations.prebid]`, and a second table would not parse. + fn settings_with_prebid(enabled: bool, timeout_ms: u32) -> Settings { + let mut settings = create_test_settings(); + settings.integrations.insert( + "prebid".to_string(), + serde_json::json!({ + "enabled": enabled, + "server_url": "https://prebid.example.com/openrtb2/auction", + "external_bundle_url": "https://assets.example.com/prebid/bundle.js", + "timeout": timeout_ms, + }), + ); + settings + } + + #[test] + fn disabling_an_integration_changes_the_fingerprint() { + // The fingerprint was `concatenated_hash(all_module_ids())` — every module + // compiled into the binary, so a constant for that binary. Turning an + // integration off changed the injected `"# .to_string(), ), - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: dispatched_auction.as_ref().map(|_| test_auction_request()), dispatched_auction, @@ -10935,7 +12169,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: Some(AuctionObservationContext::from_parts( AuctionSource::SpaNavigation, "proxy.example.com", @@ -11123,7 +12357,7 @@ mod tests { r#""# .to_string(), ), - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: Some(test_auction_request()), dispatched_auction: Some(DispatchedAuction::empty_for_test( @@ -11181,7 +12415,7 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let bids_script = r#""#; - let state = Arc::new(Mutex::new(Some(bids_script.to_string()))); + let state = AdBidsState::with_script(bids_script); let params = OwnedProcessResponseParams { template_cache_key: None, seam_ad_slots: None, @@ -11249,7 +12483,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -11360,7 +12594,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -11420,7 +12654,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index f26088db2..f3b2a5c62 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -260,8 +260,18 @@ export interface TsjsApi { * Lives in the bundle so the lifecycle is executable under test and shares * [`navGeneration`] with the SPA auction hook; `gpt_bootstrap.js` installs * a minimal fallback for pages where the bundle fails to load. + * + * `initialSlots` exists for the shared-template `` seam, which is the + * only place slot definitions arrive with the bids rather than from the head + * script. Passing them here rather than assigning `tsjs.adSlots` before the + * call puts them behind the same generation guard: an assignment made ahead + * of the guard would clobber a committed SPA navigation's slots with the SSR + * document's, and then be read by that route's `adInit()`. */ - scheduleInitialAdInit?: (initialBids?: Record) => void; + scheduleInitialAdInit?: ( + initialBids?: Record, + initialSlots?: AuctionSlot[] + ) => void; /** Read-only GPT lifecycle diagnostics API, present only in an activated tab. */ gptDiagnostics?: GptDiagnosticsApi; } diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 066bec12b..93f52f198 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -514,7 +514,15 @@ function installInitialLoadDetector(ts: TsjsApi): void { * SSR bootstrap as current. For the same reason the initial bids payload is * passed in and applied here, generation-guarded — assigning it * unconditionally at body end would clobber the live bids a faster SPA - * navigation already applied. When a navigation has committed since — or + * navigation already applied. + * + * `initialSlots` is passed in for exactly the same reason and was missing it. + * Only the shared-template `` seam sends slots — under `inline` they + * come from the head script, which runs before any navigation can commit — and + * that seam assigned `tsjs.adSlots` on the line *before* calling this. The + * guard protected the bids and `adInit()` while the assignment it was meant to + * protect had already happened, so a committed SPA navigation kept its bids and + * lost its slots. When a navigation has committed since — or * commits while the deferred callback is pending — the SSR payload is * dropped and `adInit()` is not run: running anyway would re-run the newer * route's live slots/bids, destroying and redefining that route's TS slots @@ -534,8 +542,12 @@ function installInitialLoadDetector(ts: TsjsApi): void { * holds whenever the request is actually issued. */ function installScheduleInitialAdInit(ts: TsjsApi): void { - ts.scheduleInitialAdInit = function (initialBids?: Record) { + ts.scheduleInitialAdInit = function ( + initialBids?: Record, + initialSlots?: AuctionSlot[] + ) { if ((ts.navGeneration ?? 0) !== 0) return; + if (initialSlots) ts.adSlots = initialSlots; if (initialBids) ts.bids = initialBids; const runUnlessNavigated = (): void => { if ((ts.navGeneration ?? 0) !== 0) return; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index f5c43ed7f..a0ddd6ad3 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -159,6 +159,36 @@ describe('gpt_bootstrap.js fallback', () => { expect(adInit).not.toHaveBeenCalled(); }); + it('fallback scheduler guards the SSR slot definitions with the same generation check', () => { + // The shared-template seam hands slots to the scheduler rather than assigning + // them itself, so the fallback has to honour the same guard as the bundle. If it + // applied them unconditionally, a page whose bundle failed to load would take the + // stale SSR slots over a committed navigation's. + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + const liveSlot = { + id: 'live_slot', + gam_unit_path: '/123/live', + div_id: 'div-live', + formats: [[300, 250]] as Array<[number, number]>, + }; + const ssrSlot = { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]] as Array<[number, number]>, + }; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + expect(ts.adSlots).toEqual([ssrSlot]); + + ts.adSlots = [liveSlot]; + ts.navGeneration = 1; + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + expect(ts.adSlots).toEqual([liveSlot]); + }); + it('fallback adInit defines, targets, and displays a TS slot through the command queue', () => { const mockSlot = { addService: vi.fn().mockReturnThis(), diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts index 999c60c55..830bac217 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts @@ -310,6 +310,69 @@ describe('scheduleInitialAdInit', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('applies the SSR slot definitions on the initial document', async () => { + // Under a shared-template mode the head script emits no `tsjs.adSlots`, so the + // `` seam is the only source of slot definitions. They must arrive, or + // `adInit()` iterates an empty list and the page defines no TS slots at all. + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + const ssrSlot = { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]] as Array<[number, number]>, + }; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + + expect(ts.adSlots).toEqual([ssrSlot]); + expect(ts.bids).toEqual({ ssr_slot: { hb_pb: '1.00' } }); + }); + + it('drops the SSR slot definitions when a navigation has already committed', async () => { + // The guard covered the bids and the adInit call, but the shared-template seam + // assigned `tsjs.adSlots` on the line *before* calling the scheduler — outside the + // guard entirely. A navigation that committed while the SSR document was still + // streaming therefore kept its own bids and silently lost its slots to the stale + // SSR payload, and the next `adInit()` for that route defined the wrong slots. + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/b'); + await flushAsync(); + expect(ts.navGeneration).toBe(1); + const liveSlot = { + id: 'live_slot', + gam_unit_path: '/123/live', + div_id: 'div-live', + formats: [[300, 250]] as Array<[number, number]>, + }; + ts.adSlots = [liveSlot]; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ + { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]], + }, + ]); + + expect(ts.adSlots).toEqual([liveSlot]); + + window.dispatchEvent(new Event('load')); + flushFrame(); + flushFrame(); + expect(adInit).not.toHaveBeenCalled(); + }); + it('cancels queued GPT work when a navigation commits before the command queue drains', async () => { // adInit() only queues its slot work on googletag.cmd, which drains when // GPT itself loads — possibly long after the generation check that diff --git a/scripts/c2-local-test.sh b/scripts/c2-local-test.sh index e50ea9e75..2a45e2e83 100755 --- a/scripts/c2-local-test.sh +++ b/scripts/c2-local-test.sh @@ -9,6 +9,7 @@ # # Usage: # ./scripts/c2-local-test.sh # esi mode (shared template + edge assembly) +# ./scripts/c2-local-test.sh client_fill # shared template, browser fetches its own bids # ./scripts/c2-local-test.sh inline # today's shipped behaviour, as a control # # Spike-only. Remove with the spike. @@ -16,6 +17,13 @@ set -euo pipefail MODE="${1:-esi}" +case "$MODE" in + inline | client_fill | esi) ;; + *) + echo "Unknown mode '$MODE'. Use one of: inline, client_fill, esi." >&2 + exit 1 + ;; +esac REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" WORK="$(mktemp -d)" ORIGIN_PORT="${ORIGIN_PORT:-9099}" @@ -81,10 +89,38 @@ cat > "$WORK/origin.py" <stub creative", + "w": 728, + "h": 90, + } + ], + } + ], +} + PAGE = b""" Stub article @@ -125,7 +161,7 @@ class H(BaseHTTPRequestHandler): if n: self.rfile.read(n) time.sleep($BID_DELAY) - self._send(json.dumps({"id": "stub", "seatbid": []}).encode(), "application/json") + self._send(json.dumps(BID_RESPONSE).encode(), "application/json") def log_message(self, fmt, *args): print("origin: " + fmt % args, flush=True) @@ -222,6 +258,16 @@ info "Running assertions (mode: $MODE)" BEFORE=$(origin_gets) R1=$(req "$WORK/r1.html") R2=$(req "$WORK/r2.html") + +# Content assertions must never run against compressed bytes. `inline` responses stay +# gzipped end to end — only the shared path decodes, because its seam split is textual — +# and `grep` over a gzip stream matches nothing, which reads as a pass for every +# "must not contain" check and as a silent failure for every "must contain" one. +# Decode a copy and assert against that, in both modes. +SERVED="$WORK/r2.served.html" +if ! gzip -dc "$WORK/r2.html" > "$SERVED" 2>/dev/null; then + cp "$WORK/r2.html" "$SERVED" +fi AFTER=$(origin_gets) FETCHES=$((AFTER - BEFORE)) @@ -231,47 +277,109 @@ read -r TTFB2 TOTAL2 CODE2 <<< "$R2" check "first request returns 200" "$CODE1" "200" check "second request returns 200" "$CODE2" "200" +# The bid the stub origin returns, bucketed and then escaped the way the seam escapes +# it. Asserted in both modes: a shared-mode failure that inline shares would otherwise +# read as "the fixture never bids" rather than "the seam drops bids". +WINNING_BID='\"hb_pb\":\"4.25\"' +# Must stay in step with `SEAM_BIDS_MARKER` in publisher.rs. An inert HTML comment, +# not an esi:include — nothing parses ESI on the render path any more. +SEAM_MARKER='' + +# Shared by every mode that stores a template, so `esi` and `client_fill` cannot drift +# apart on the two properties that have nothing to do with the seam. +check_hit_is_private() { + local hdrs + hdrs=$(curl -s -D- -o /dev/null -H "Host: ts.example.com" \ + -H "Accept-Encoding: gzip" \ + -H "sec-fetch-dest: document" -H "sec-fetch-mode: navigate" \ + "http://127.0.0.1:$TS_PORT/article") + check "cache hit is not shared-cacheable" \ + "$(echo "$hdrs" | grep -ci 'cache-control: private, no-store' || true)" "1" +} + +check_post_reaches_origin() { + local before + before=$(grep -c "POST /article" "$WORK/origin.log" || true) + curl -s -o /dev/null -X POST -d 'x=1' -H "Host: ts.example.com" \ + -H "Accept-Encoding: gzip" \ + "http://127.0.0.1:$TS_PORT/article" + check "a POST still reaches the origin" \ + "$(( $(grep -c "POST /article" "$WORK/origin.log" || true) - before ))" "1" +} + if [ "$MODE" = "inline" ]; then check "inline fetches the origin every time" "$FETCHES" "2" check "inline writes no shared template" \ "$(grep -c 'c2_template_cache stored' "$WORK/viceroy.log" || true)" "0" + check "inline delivers the winning bid" \ + "$(grep -cF "$WINNING_BID" "$SERVED" || true)" "1" +elif [ "$MODE" = "client_fill" ]; then + # The assertion this mode never had. `client_fill` stored a template on every + # request and read one back on none of them: both hit paths demanded a seam marker, + # and this mode emits none by design, so each hit was discarded as transform drift + # and refetched the origin. Its caching had never worked. + check "second request is served from cache" "$FETCHES" "1" + # Asserted on the *distinct* values rather than on a line count: viceroy emits each + # log line twice (once to the log endpoint, once to stdout), so counting lines + # measures the logger. One distinct value means every store agreed, and `false` is + # the value this mode must produce — a `true` here would mean it had grown a seam. + check "the stored template has no seam marker, by design" \ + "$(grep -ohE 'seam marker present: [a-z]+' "$WORK/viceroy.log" | sort -u | tr '\n' '/')" \ + "seam marker present: false/" + check "no seam marker reaches the browser" \ + "$(grep -cF "$SEAM_MARKER" "$SERVED" || true)" "0" + # The mode's defining property: the browser fetches its own bids, so the server + # splices none. A page carrying the server's bucketed price would mean this mode had + # quietly become `esi`. + check "the server splices no bids" \ + "$(grep -cF "$WINNING_BID" "$SERVED" || true)" "0" + check "the server schedules no initial adInit" \ + "$(grep -c 'scheduleInitialAdInit' "$SERVED" || true)" "0" + # `root_auction_is_useful` refuses to dispatch here: an auction nothing reads would + # bill the SSPs and hold the response for its full budget with no consumer. + check "no root auction is dispatched" \ + "$(grep -c "POST /bid" "$WORK/origin.log" || true)" "0" + check_hit_is_private + check_post_reaches_origin else check "second request is served from cache" "$FETCHES" "1" - check "no unresolved esi:include reaches the browser" \ - "$(grep -c 'esi:include' "$WORK/r2.html" || true)" "0" + check "no unresolved seam marker reaches the browser" \ + "$(grep -cF "$SEAM_MARKER" "$SERVED" || true)" "0" check "a bids script is present" \ - "$(grep -c 'window.tsjs' "$WORK/r2.html" || true)" "1" + "$(grep -c 'window.tsjs' "$SERVED" || true)" "1" # `window.tsjs` alone passes while initial ads are dead: shared modes suppress the head # slot script, so if the seam does not carry slots, `adSlots` stays `[]` and `adInit` # defines nothing. This harness passed green through exactly that bug. + # The slots ride the scheduler call (`s(b,a)`) rather than a bare assignment, so the + # navigation-generation guard covers them; `var a=JSON.parse(...)` is where they land. check "the seam carries slot definitions, not just bids" \ - "$(grep -c 'adSlots=JSON.parse' "$WORK/r2.html" || true)" "1" + "$(grep -c 'var a=JSON.parse' "$SERVED" || true)" "1" + check "the slot definitions reach the guarded scheduler" \ + "$(grep -cF 's(b,a)' "$SERVED" || true)" "1" check "the slot definitions are populated, not an empty array" \ - "$(grep -c 'adSlots=JSON.parse("\[\]")' "$WORK/r2.html" || true)" "0" - - HDRS=$(curl -s -D- -o /dev/null -H "Host: ts.example.com" \ - -H "Accept-Encoding: gzip" \ - -H "sec-fetch-dest: document" -H "sec-fetch-mode: navigate" \ - "http://127.0.0.1:$TS_PORT/article") - check "cache hit is not shared-cacheable" \ - "$(echo "$HDRS" | grep -ci 'cache-control: private, no-store' || true)" "1" - - POSTS_BEFORE=$(grep -c "POST /article" "$WORK/origin.log" || true) - curl -s -o /dev/null -X POST -d 'x=1' -H "Host: ts.example.com" \ - -H "Accept-Encoding: gzip" \ - "http://127.0.0.1:$TS_PORT/article" - check "a POST still reaches the origin" \ - "$(( $(grep -c "POST /article" "$WORK/origin.log" || true) - POSTS_BEFORE ))" "1" + "$(grep -c 'var a=JSON.parse("\[\]")' "$SERVED" || true)" "0" + # The assertion the harness was missing entirely. `window.tsjs` and populated slots + # both pass on a page whose bids are `{}` — which is what shared modes served, on + # every request, for as long as this file has existed. + check "the seam carries a real bid, not an empty map" \ + "$(grep -cF 'var b=JSON.parse("{}")' "$SERVED" || true)" "0" + check "the winning bid's bucketed price reaches the reader" \ + "$(grep -cF "$WINNING_BID" "$SERVED" || true)" "1" + + check_hit_is_private + check_post_reaches_origin fi -if [ "$MODE" != "inline" ]; then +if [ "$MODE" = "esi" ]; then info "Where the marker actually lives" echo " The cached template (the shared copy — has a hole where bids go):" grep -oE "c2_template_cache stored [0-9]+ bytes \(seam marker present: [a-z]+\)" \ "$WORK/viceroy.log" | sort -u | sed 's/^/ /' echo echo " What the reader receives (hole filled, no marker):" - grep -oE "esi:include|window\.tsjs" "$WORK/r2.html" | sort | uniq -c | sed 's/^/ /' + printf ' %d seam marker(s), %d window.tsjs\n' \ + "$(grep -cF "$SEAM_MARKER" "$SERVED" || true)" \ + "$(grep -c 'window\.tsjs' "$SERVED" || true)" cat <<'EOF' The marker is never visible in page source, in any mode. It exists only inside @@ -350,6 +458,16 @@ if [ "$MODE" = "inline" ]; then check "inline delivers the article before the auction resolves" \ "$(awk -v f="$FIRST_BODY" -v c="$COMPLETE" 'BEGIN { print (f < c / 3) ? "yes" : "no" }')" \ "yes" +elif [ "$MODE" = "client_fill" ]; then + check "the origin fetch stays compressed" \ + "$(grep -c 'served PLAINTEXT' "$WORK/origin.log" || true)" "0" + # There is no seam to hold, so there is nothing to hold *for*: the whole response + # lands well inside the bid endpoint's delay. Stated as a bound on the total rather + # than as a ratio, because with no auction the first body byte and the last arrive + # together and `first < complete / 3` would be meaningless here. + check "the whole response lands without waiting for an auction" \ + "$(awk -v c="$COMPLETE" -v d="$BID_DELAY" 'BEGIN { print (c < d * 500) ? "yes" : "no" }')" \ + "yes" else # The property the unit tests cannot reach: in-process there is no bid provider, so # there is no auction to wait on and reordering the stream is unobservable. Here the From 4ec910484920fe169062999753c947c890150caa Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 12 Aug 2026 14:02:39 +0530 Subject: [PATCH 218/395] Plan the #1009 main merge and ESI hardening --- .../2026-08-12-1009-esi-merge-hardening.md | 220 ++++++++++++++++++ ...6-08-12-1009-esi-merge-hardening-design.md | 170 ++++++++++++++ 2 files changed, 390 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md create mode 100644 docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md diff --git a/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md new file mode 100644 index 000000000..7555db47b --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md @@ -0,0 +1,220 @@ +# #1009 ESI Merge and Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:executing-plans` to implement +> this plan task-by-task. This plan is intentionally executed inline because the operator +> explicitly prohibited subagents. + +**Goal:** Merge current `main` and make the opt-in ESI byte-seam/shared-template path correct, +private, cache-semantic, compressed, observable, and operationally reversible. + +**Architecture:** Fastly Core Cache holds identity-encoded reader-neutral templates behind a +transaction acquired before origin work. Every request assembles its own slots and structured bid +map at an exact inert seam, encodes the result for that client, and receives a final immutable +private/no-store policy. + +**Tech Stack:** Rust 1.95, Fastly Compute/Core Cache, `edgezero_core` HTTP types, `lol_html`, +TypeScript/Vitest, Viceroy, shell harness. + +--- + +### Task 1: Merge live main and preserve auction contracts + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/integrations/gpt_diagnostics.rs` +- Modify: `crates/trusted-server-js/lib/src/core/types.ts` +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Test: adjacent Rust and Vitest modules + +- [ ] Merge `origin/main` with `git merge --no-ff origin/main`. +- [ ] Resolve the `AdBidsState`/`write_bids_to_state` conflict by building one structured map with + `auction_id`, storing both map and script, and returning its delivered slot IDs. +- [ ] Add/adjust tests proving ESI and inline retain `hb_auction_id`, APS renderer metadata, and + delivered-winner attribution. +- [ ] Run the focused Rust and GPT tests. +- [ ] Complete the merge commit. + +### Task 2: Remove mechanisms outside the approved ESI byte-seam design + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Delete: `crates/trusted-server-core/src/platform/template_assembly.rs` +- Modify: `crates/trusted-server-core/src/platform/mod.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Delete: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-adapter-fastly/Cargo.toml` +- Modify: `Cargo.lock` + +- [ ] Update mode tests to specify only `inline` and `esi`; watch the old client-fill expectations + fail or stop compiling. +- [ ] Remove `ClientFill`, executable fragment serialization, assembler traits/registration, and + the `esi` crate. +- [ ] Update comments to call the production path byte-seam assembly. +- [ ] Run focused configuration, publisher, and Fastly adapter tests. +- [ ] Commit the scope cleanup. + +### Task 3: Canonicalize and bound the template key + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] Write failing tests for absent versus empty `Vary`, repeated raw values, invalid configured + names, punctuation-colliding purge URLs, changed origin host override, and changed creative + configuration. +- [ ] Replace string pairs with a typed canonical `Vary` value preserving presence and all bytes. +- [ ] Hash a length-prefixed canonical key and hash the URL-specific surrogate key. +- [ ] Include publisher origin identity and the complete template-shaping fingerprint. +- [ ] Run focused key/configuration tests and commit. + +### Task 4: Enforce request and origin cache semantics + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/Cargo.toml` if HTTP-date parsing needs a direct dependency + +- [ ] Write failing tests for response `max-age=0`, positive max age, repeated/malformed cache + directives, `Age` exhaustion, expired/malformed `Expires`, missing freshness, invalid `Vary`, + and request no-cache/no-store/range/conditional bypasses. +- [ ] Add a typed cache eligibility result carrying the positive remaining TTL. +- [ ] Parse relevant response directives fail-closed and cap, never extend, origin freshness. +- [ ] Add request-side bypass classification before lookup. +- [ ] Make unsupported/backend-failed cache lookups fall back to inline processing on non-Fastly + adapters rather than buffering a cacheless ESI path. +- [ ] Run focused eligibility tests and commit. + +### Task 5: Move request collapse before the origin fetch + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` + +- [ ] Write a failing cache test showing two concurrent cold requests currently perform two + origin fetches/transforms. +- [ ] Introduce a lookup outcome with an opaque insert reservation and explicit cancellation. +- [ ] Implement Fastly `Transaction::lookup` before origin work and consume/cancel its obligation + on every exit path. +- [ ] Ensure invalid fresh entries become replaceable rather than causing repeated refetches. +- [ ] Run focused Core Cache/Viceroy tests and commit. + +### Task 6: Make privacy and policy-header parity final + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/response_privacy.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` tests + +- [ ] Write failing tests for repeated CSP/CSP-Report-Only, omitted COOP/COEP/CORP/HSTS/Link, + unknown cached header metadata, duplicate required metadata fields, and a late integration + changing `Cache-Control` to public. +- [ ] Capture all ordered values, expand the safe allowlist, and decode metadata strictly. +- [ ] Replay with `append`, then apply the assembled-response privacy policy last. +- [ ] Preserve and reassert private/no-store after request-filter effects in Fastly's final send. +- [ ] Run focused header/privacy tests and commit. + +### Task 7: Bypass shared templates for request-private diagnostics + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] Write a failing warm-cache test activated by diagnostics query and another by diagnostics + cookie. +- [ ] Make `requires_private_no_store()` a lookup/store disqualifier. +- [ ] Verify ordinary diagnostics-disabled requests still hit C2. +- [ ] Run focused diagnostics/C2 tests and commit. + +### Task 8: Re-encode assembled responses for the reader + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] Write failing cold/warm tests requiring gzip/br clients to receive a matching encoded body + and proving reader encoding no longer partitions the stored template. +- [ ] Use one canonical compressed `Accept-Encoding` offer for every shared-template origin miss. +- [ ] Carry the selected response encoding separately from identity template metadata. +- [ ] Encode buffered assembly after splicing and stream hit prefix/seam/suffix through one encoder. +- [ ] Handle `identity;q=0` without serving an unacceptable representation. +- [ ] Emit the correct `Vary: Accept-Encoding` response semantics after final encoding. +- [ ] Run focused compression tests and commit. + +### Task 9: Make marker failures safe + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` + +- [ ] Write failing tests for HTML with no explicit ``, a publisher-authored marker + collision, and a corrupt cached marker. +- [ ] Record/validate a schema-bound seam location or use a collision-resistant marker contract. +- [ ] Cancel storage and fall back safely when the optimization cannot produce one seam. +- [ ] Run focused miss/hit assembly tests and commit. + +### Task 10: Add operational observability and harden the harness + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` +- Modify: `scripts/c2-local-test.sh` +- Modify: `.github/workflows/test.yml` + +- [ ] Write failing tests for distinct backend-error versus not-found status and C2 response-state + reporting. +- [ ] Preserve backend errors and emit bounded C2 status without exposing key material. +- [ ] Change the harness to operate on a temporary manifest and fail on missing/non-numeric probe + output or empty response bodies. +- [ ] Test both cold and warm integrity and execute the generated scheduler payload contract. +- [ ] Add the ESI harness to CI where Viceroy prerequisites are available. +- [ ] Run shell syntax/static checks and commit. + +### Task 11: Document configuration, semantics, and rollback + +**Files:** + +- Modify: `trusted-server.example.toml` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md` +- Modify: `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md` +- Modify: relevant #1009 findings documents + +- [ ] Document that `esi` means Fastly C2 plus byte-seam assembly, not parser execution or final + HTTP shared caching. +- [ ] Document `template_cache_vary`, cookie independence, freshness, metrics, purge, rollback + ordering, and limitations on non-Fastly adapters. +- [ ] Close or supersede stale spike checkboxes and remove claims contradicted by the final code. +- [ ] Run docs format/build and commit. + +### Task 12: Full verification + +**Files:** none expected beyond fixes discovered by verification + +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run all four adapter test aliases and the parity suite. +- [ ] Run all six clippy aliases. +- [ ] Build the Fastly release WASM. +- [ ] Run JS tests, build, and format under pinned Node 24.12.0. +- [ ] Run docs format/build. +- [ ] Run `scripts/c2-local-test.sh esi` and `inline` if the environment exposes the required + local certificate store; otherwise report the exact environment blocker. +- [ ] Run `git diff --check`, inspect the merge graph, and confirm the worktree contains only + intended changes. diff --git a/docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md b/docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md new file mode 100644 index 000000000..c113f9675 --- /dev/null +++ b/docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md @@ -0,0 +1,170 @@ +# #1009 ESI Merge and Hardening Design + +**Date:** 2026-08-12 +**Status:** Approved for implementation +**Branch:** `1009-esi-cacheable-root-spec` + +## Goal + +Merge current `main` into the #1009 branch and leave the opt-in +`creative_opportunities.assembly_mode = "esi"` path safe, testable, and ready for a +controlled Fastly rollout. The setting keeps its existing operator spelling, while the +render path uses an inert marker and an exact byte seam rather than the `esi` parser. + +The result caches only a reader-neutral transformed template in Fastly Core Cache. The +assembled navigation response remains request-specific and must always leave Trusted Server +as `private, no-store`. + +## Non-goals + +- Restoring a top-level HTTP `x-cache: HIT`. Compute still executes for every assembled + navigation; the hit is in the internal C2 template cache. +- Shipping `client_fill`. That comparison arm is incomplete and outside the requested ESI + scope. +- Retaining a general-purpose ESI parser. One known marker does not justify a second HTML + parser or its dependency surface. +- Changing GAM line-item, creative-selection, or win-notification behavior. + +## Merge contract + +`origin/main` is merged with a normal merge commit, never rebased. Conflict resolution in the +auction state must preserve both sides: + +- the branch's structured bid map used by byte-seam assembly; +- main's per-auction `hb_auction_id`; +- main's delivered-winner slot set used by auction telemetry; +- APS typed-renderer metadata carried on winning bids; +- the current GPT slot-handoff and duplicate-request protections; +- the generation-zero guard that applies ESI slots and bids atomically. + +Tests must cover the same non-empty winning bid through inline and ESI output after the merge. + +## Shared-cache contract + +### Transaction starts before origin work + +The platform cache lookup returns one of three outcomes: + +1. a usable template hit; +2. an insert reservation owned by this request; +3. unsupported/backend failure, which bypasses the optimization. + +On Fastly, `Transaction::lookup` begins before the publisher origin request. A miss owner carries +an opaque reservation through the origin fetch and transform. It either inserts the neutral +template or explicitly cancels the obligation when the response is ineligible or processing +fails. Concurrent requests wait on the transaction and reuse the inserted object instead of each +fetching and transforming the origin. + +### Key is canonical and bounded + +The key hashes a length-prefixed canonical representation rather than sending raw URL/header +values to Core Cache. Inputs include: + +- schema version and assembly mode; +- reader-facing scheme/host and full target URI; +- publisher origin URL and host-header override; +- a digest of all template-shaping configuration and the TSJS bundle; +- each configured `Vary` header with an explicit absent/present distinction and every raw field + value in wire order. + +Configured `Vary` names are validated as HTTP header names and deduplicated. Invalid response +`Vary` values fail closed. Per-URL purge keys use a digest too, avoiding punctuation collisions +and platform length limits. + +### Origin freshness is authoritative + +C2 never invents freshness. Eligibility requires a positive remaining shared lifetime derived +from the origin's cache directives. `private`, `no-store`, `no-cache`, zero freshness, malformed +directives, or already-consumed freshness all bypass storage. The stored max age is capped by the +short operator safety ceiling and reduced by `Age`. + +Requests carrying `Cache-Control: no-cache`/`max-age=0`, `Pragma: no-cache`, range headers, or +conditional validators bypass C2 lookup. Authentication, cookie independence, and diagnostics +privacy remain request-side gates. + +`no-cache` forces a fresh origin read but may replace C2 with the newly validated response; +request `no-store` forbids both lookup and insertion. On adapters without a shared-template cache, +or when the cache backend fails before reservation, `esi` degrades to the existing inline path so +an optimization outage does not add full-document buffering. + +## Response assembly and representation + +Templates are stored as identity bytes because marker validation and splitting are textual. ESI +misses offer a fixed, supported compressed `Accept-Encoding` set to the origin, independent of the +reader, so every reader selects the same upstream representation before it is decoded. The final +assembled response is encoded for the current client's accepted representation: + +- buffered misses assemble first and then encode; +- Fastly hits encode the prefix, request seam, and suffix through one streaming encoder; +- the template key does not vary on `Accept-Encoding`, because the stored representation is + always identity and the upstream offer is canonical. + +Missing or repeated markers are optimization failures, not publisher outages. An invalid hit is +discarded and replaced through the transactional miss path. A newly transformed document without +a normal body-close seam receives a collision-resistant fallback marker at document end; if a +valid seam still cannot be established, the document is not stored and is delivered safely rather +than turning an origin 200 into a Trusted Server 5xx. + +## Privacy and response metadata + +The decision that a response must remain private is carried to the final Fastly send. Request +filter effects and operator headers run first; then Trusted Server reapplies `private, no-store` +and removes every CDN-specific cache directive. + +Cached policy metadata is strictly decoded: + +- only allowlisted names are accepted; +- required fields may appear exactly once; +- repeated policy values are preserved in order; +- invalid metadata is a cache miss, never a partially reconstructed response. + +Warm responses preserve repeated CSP/CSP-Report-Only and other per-document security/performance +headers such as COOP, COEP, CORP, HSTS, Origin-Agent-Cluster, reporting headers, and `Link`. +Privacy is stamped after replay so metadata cannot override it. + +A nonce-bearing CSP is not automatically rejected. If an origin explicitly marks the matching +HTML and CSP response shareable, sharing that exact header/body pair is already part of the +origin's cache contract. C2 must not extend its freshness beyond that contract. + +## Scope cleanup + +The implementation removes: + +- `AssemblyMode::ClientFill` and its harness arm; +- `PlatformTemplateAssembler` and the Fastly assembler registration; +- `esi_assembly.rs` and the `esi` dependency; +- the unused executable `format=fragment` response. + +The public string `assembly_mode = "esi"` remains for operator continuity. Documentation describes +it as edge byte-seam assembly and makes its Fastly-only cache acceleration explicit. + +## Observability and operations + +Every request reports a distinct C2 state: bypass, hit, cold miss/reservation, invalid entry, +backend error, store, or cancelled reservation. Backend errors are not collapsed into ordinary +misses. A response header suitable for canary inspection is added without exposing key material. + +Rollback is: + +1. set `assembly_mode = "inline"`; +2. remove the new keys before rolling back to a binary whose configuration uses + `deny_unknown_fields`; +3. purge the `ts-template` surrogate key or wait for the bounded origin-derived TTL. + +The local harness works entirely from a temporary Fastly manifest, fails closed when timing data +is missing, verifies cold and warm response bodies, and executes enough of the generated seam +contract to prove populated slots and bids reach the guarded scheduler. CI runs it in ESI mode. + +## Verification + +Required gates after implementation: + +- focused red/green tests for each defect; +- `cargo test-fastly`, `cargo test-axum`, `cargo test-cloudflare`, `cargo test-spin`; +- all six target-matched clippy aliases; +- Fastly release WASM build; +- JS tests/build/format under Node 24.12.0; +- documentation format/build; +- cross-adapter parity suite; +- local C2 harness in `esi` and `inline` modes when Viceroy can access the local certificate + store. From e25552128e7841b882baee2ab1c8a24a66abbb5c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 12 Aug 2026 14:22:22 +0530 Subject: [PATCH 219/395] Remove unused parser and client-fill spike paths --- Cargo.lock | 112 +--- .../trusted-server-adapter-fastly/Cargo.toml | 1 - .../trusted-server-adapter-fastly/src/app.rs | 1 - .../src/esi_assembly.rs | 466 --------------- .../trusted-server-adapter-fastly/src/main.rs | 1 - .../src/template_cache.rs | 8 +- .../src/creative_opportunities.rs | 29 +- .../trusted-server-core/src/html_processor.rs | 3 +- .../trusted-server-core/src/platform/mod.rs | 4 - .../src/platform/template_assembly.rs | 87 --- .../src/platform/template_cache.rs | 2 +- .../trusted-server-core/src/platform/types.rs | 42 -- crates/trusted-server-core/src/publisher.rs | 542 +++--------------- scripts/c2-local-test.sh | 46 +- 14 files changed, 121 insertions(+), 1223 deletions(-) delete mode 100644 crates/trusted-server-adapter-fastly/src/esi_assembly.rs delete mode 100644 crates/trusted-server-core/src/platform/template_assembly.rs diff --git a/Cargo.lock b/Cargo.lock index 911acebda..cb42ad315 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,7 +146,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom 7.1.3", + "nom", "num-traits", "rusticata-macros", "thiserror 1.0.69", @@ -254,15 +254,6 @@ dependencies = [ "tungstenite", ] -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -582,18 +573,7 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures 0.2.17", -] - -[[package]] -name = "chacha20" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.1", + "cpufeatures", ] [[package]] @@ -603,7 +583,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20 0.9.1", + "chacha20", "cipher", "poly1305", "zeroize", @@ -936,15 +916,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -1070,7 +1041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", @@ -1215,7 +1186,7 @@ checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" dependencies = [ "asn1-rs", "displaydoc", - "nom 7.1.3", + "nom", "num-bigint", "num-traits", "rusticata-macros", @@ -1719,27 +1690,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "esi" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e384a711090b57e3dd20080915935607078ab0b43d49575994b44dd36956f84" -dependencies = [ - "atoi", - "base64", - "bytes", - "chrono", - "fastly", - "html-escape", - "log", - "md5", - "nom 8.0.0", - "percent-encoding", - "rand 0.10.2", - "regex", - "thiserror 2.0.18", -] - [[package]] name = "etcetera" version = "0.10.0" @@ -2084,7 +2034,6 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "rand_core 0.10.1", ] [[package]] @@ -2239,12 +2188,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "html-escape" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9356095b4b41197bba32173600e1582792cda618f65d12f68e2e77d273413c5" - [[package]] name = "html5ever" version = "0.35.0" @@ -2971,12 +2914,6 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" -[[package]] -name = "md5" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" - [[package]] name = "memchr" version = "2.8.2" @@ -3047,15 +2984,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nom" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" -dependencies = [ - "memchr", -] - [[package]] name = "num" version = "0.4.3" @@ -3549,7 +3477,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures 0.2.17", + "cpufeatures", "opaque-debug", "universal-hash", ] @@ -3847,17 +3775,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" -dependencies = [ - "chacha20 0.10.1", - "getrandom 0.4.3", - "rand_core 0.10.1", -] - [[package]] name = "rand_chacha" version = "0.3.1" @@ -3896,12 +3813,6 @@ dependencies = [ "getrandom 0.3.4", ] -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - [[package]] name = "rcgen" version = "0.13.2" @@ -4168,7 +4079,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -4583,7 +4494,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest 0.10.7", ] @@ -4595,7 +4506,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest 0.9.0", "opaque-debug", ] @@ -4607,7 +4518,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest 0.10.7", ] @@ -5366,7 +5277,6 @@ dependencies = [ "edgezero-adapter-fastly", "edgezero-core", "error-stack", - "esi", "fastly", "fern", "futures", @@ -6416,7 +6326,7 @@ dependencies = [ "data-encoding", "der-parser", "lazy_static", - "nom 7.1.3", + "nom", "oid-registry", "ring", "rusticata-macros", diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index b36b43f86..78f56bc2d 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -19,7 +19,6 @@ derive_more = { workspace = true } edgezero-adapter-fastly = { workspace = true, features = ["fastly"] } edgezero-core = { workspace = true } error-stack = { workspace = true } -esi = "0.7" fastly = { workspace = true } fern = { workspace = true } futures = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index cc9e575ee..6c54184bd 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -260,7 +260,6 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime // Spike-only (#1009). Constructed unconditionally, but only read when the // assembly mode is a shared-template one — which defaults to Inline, so this // is inert until an operator opts in. - .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) .template_cache(Arc::new(crate::template_cache::FastlyTemplateCache::new( crate::template_cache::TEMPLATE_CACHE_TTL, ))) diff --git a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs deleted file mode 100644 index 5959904ba..000000000 --- a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs +++ /dev/null @@ -1,466 +0,0 @@ -//! Edge assembly: splice the per-user fragment into the shared template (arm A3). -//! -//! This is the step that distinguishes A3 from A2. Both serve the same C2 template; -//! A2 has the browser fetch the fragment, A3 resolves it here, before the bytes leave -//! the edge. Whether that difference is worth a Fastly-only rendering path is the -//! question the spike exists to answer. -//! -//! **The fragment is resolved before this runs, not by it.** `esi`'s dispatcher is -//! synchronous, and this codebase's fragment producer is `async`; calling it from -//! inside the dispatcher would mean a nested executor, which panics. The way out is -//! [`PendingFragmentContent::CompletedRequest`], which lets the dispatcher hand back a -//! response that was already built. So the caller runs the auction in the normal async -//! flow and passes the bytes in, and the dispatcher never performs I/O at all — no -//! subrequest, no backend, no self-call. -//! -//! Spike-only. Remove with the spike. - -use esi::{CacheConfig, Configuration, DcaMode, PendingFragmentContent, Processor}; -use fastly::Response; -use fastly::http::StatusCode; -use std::io::Cursor; -use trusted_server_core::platform::{PlatformTemplateAssembler, TemplateAssemblyError}; - -/// The processor configuration, with every safety-relevant field stated. -/// -/// Not `Configuration::default()`. Two of these settings fail **open**, and this is a -/// pre-1.0 crate whose defaults can move in a patch release — a comment saying "the -/// default is already what we want" would be an assumption rechecked by nobody. -/// -/// The two that matter: -/// -/// - **`is_includes_cacheable` defaults to `true`.** Fragments here carry one visitor's -/// bids. Letting the ESI layer cache them is precisely the per-user leak this whole -/// design exists to prevent, and it would happen silently on a cache hit. -/// - **`default_dca` / `inherit_parent_dca`** decide whether fragment bytes are -/// re-parsed as ESI. Our fragment is a `"; - - fn template_with_include() -> String { - format!("
{ESI_BIDS_INCLUDE}") - } - - #[test] - fn the_seams_own_marker_is_resolved() { - // The processor must resolve a well-formed include. This is the crate's - // contract, not the seam's — the seam no longer emits ESI at all. - let assembled = assemble(&template_with_include(), FRAGMENT).expect("should assemble"); - - assert!( - assembled.contains(FRAGMENT), - "the fragment must reach the document: {assembled}" - ); - assert!( - !assembled.contains("esi:include"), - "no unresolved include may survive: {assembled}" - ); - } - - #[test] - fn the_fragment_lands_where_the_marker_was() { - // Position matters: the script reads slots defined earlier in the document, so - // an assembler that appended instead of substituting would produce a page that - // parses and does nothing. - let assembled = assemble(&template_with_include(), FRAGMENT).expect("should assemble"); - - let slot = assembled.find("id=\"slot\"").expect("slot should survive"); - let script = assembled - .find(FRAGMENT) - .expect("fragment should be present"); - let body_close = assembled - .find("") - .expect("body close should survive"); - - assert!(slot < script, "the fragment must follow the slot markup"); - assert!(script < body_close, "the fragment must precede ``"); - } - - #[test] - fn a_document_without_an_include_is_returned_unchanged() { - // Inline mode's documents pass through this path only if something is - // misrouted, and a mangled document would be a far worse failure than a no-op. - let plain = "

no includes here

"; - - assert_eq!( - assemble(plain, FRAGMENT).expect("should assemble"), - plain, - "a template with nothing to splice must be byte-identical" - ); - } - - #[test] - fn an_empty_fragment_still_removes_the_marker() { - // The empty-bids case is normal, not exceptional: an auction that returned - // nothing still has to produce a document with no `esi:include` left in it, or - // the browser renders the raw tag as text. - let assembled = assemble(&template_with_include(), "").expect("should assemble"); - - assert!( - !assembled.contains("esi:include"), - "an empty fragment must still consume the marker: {assembled}" - ); - assert!(assembled.contains(""), "the document must survive"); - } - - #[test] - fn fragment_caching_is_off_because_its_default_leaks() { - // `is_includes_cacheable` defaults to `true`. A fragment here carries one - // visitor's bids, so caching it serves those bids to the next visitor. This is - // the single most consequential line in the module and the default is wrong, - // which is why it is asserted rather than trusted. - let config = assembly_configuration(); - - assert!( - !config.cache.is_includes_cacheable, - "per-user fragments must never be cached by the ESI layer" - ); - assert!( - config.cache.includes_force_ttl.is_none(), - "force_ttl caches everything, ignoring private/no-store/Set-Cookie" - ); - } - - #[test] - fn fragment_bytes_are_never_reparsed_as_esi() { - // The fragment is a script built from auction data. Re-parsing it as ESI would - // let bid content act as markup instructions. - let config = assembly_configuration(); - - assert_eq!(config.default_dca, DcaMode::None); - assert!(!config.inherit_parent_dca); - } - - #[test] - fn the_processor_emits_no_cache_headers_of_its_own() { - // The publisher path sets `private, no-store` before any body byte is written, - // and on this adapter headers cannot change once streaming starts. A - // Cache-Control derived from include TTLs would contradict it, and the - // contradiction would favour caching. - let config = assembly_configuration(); - - assert!(!config.cache.is_rendered_cacheable); - assert!(!config.cache.rendered_cache_control); - assert!(!config.enable_edge_control); - } - - #[test] - fn a_nested_include_inside_a_fragment_is_not_followed() { - // Depth is capped at 1, and the fragment is not parsed as ESI, so a fragment - // that happened to contain an include tag must be spliced as text rather than - // dispatched. Otherwise auction data could drive fragment requests. - let fragment = ""; - let assembled = assemble(&template_with_include(), fragment).expect("should assemble"); - - assert!( - assembled.contains("/evil"), - "the inner tag must survive as text rather than being resolved: {assembled}" - ); - } - - #[test] - fn react_suspense_markers_and_inline_scripts_survive() { - // The document this runs over is the publisher's entire page, not an ESI - // template. React marks Suspense boundaries with HTML comments — ``, - // ``, ``, `` — and hydration fails if they are altered - // or dropped. Next.js also embeds inline scripts full of escaped JSON. - // - // Nothing previously checked that assembly is byte-faithful to any of it. Every - // test used a five-line fixture. - let document = format!( - "\ - \ - \ - \ - \ -

copy

{ESI_BIDS_INCLUDE}" - ); - - let assembled = assemble(&document, FRAGMENT).expect("should assemble"); - - for marker in ["", "", "", ""] { - assert!( - assembled.contains(marker), - "React Suspense marker {marker} must survive assembly, or hydration \ - fails: {assembled}" - ); - } - assert!( - assembled.contains("a\\u003eb & c"), - "inline script content must be byte-faithful: {assembled}" - ); - // Everything except the marker substitution must be untouched. - assert_eq!( - assembled, - document.replace(ESI_BIDS_INCLUDE, FRAGMENT), - "assembly must change nothing but the seam" - ); - } - - #[test] - fn a_large_realistic_document_survives_byte_identically() { - // The real document is ~690KB of publisher HTML. Every other test here uses a - // handful of lines, and `Configuration` has a `chunk_size` — so a parser that is - // faithful on a small input can still corrupt one that spans many chunks. - // - // Observed on a real deployment: the cache-miss path, which runs this parser over - // the whole document, produced a broken page, while the cache-hit path, which - // does a plain byte split, produced a working one. This is that difference under - // test. - let mut document = String::from("t"); - for i in 0..6000 { - document.push_str(&format!( - "

copy {i} <tag>

\ -
" - )); - } - document.push_str(&format!("{ESI_BIDS_INCLUDE}")); - assert!( - document.len() > 600_000, - "fixture must be realistically large, got {}", - document.len() - ); - - let assembled = assemble(&document, FRAGMENT).expect("should assemble"); - - assert_eq!( - assembled.len(), - document.len() - ESI_BIDS_INCLUDE.len() + FRAGMENT.len(), - "assembled length must differ from the source by exactly the seam swap" - ); - assert_eq!( - assembled, - document.replace(ESI_BIDS_INCLUDE, FRAGMENT), - "a large document must survive byte-identically apart from the seam" - ); - } - - #[test] - fn a_document_larger_than_the_real_page_is_not_truncated() { - // The real page is ~1.4MB decoded. A deployment served 691704 bytes of it and - // the browser showed an error boundary — the document was cut roughly in half. - // The cache-hit path, which does a plain byte split, served the same page - // correctly, so the loss is in this parser and only shows up above some size the - // 600KB test does not reach. - let mut document = String::from("t"); - for i in 0..14000 { - document.push_str(&format!( - "

copy {i} <tag>

\ -
" - )); - } - document.push_str(&format!("{ESI_BIDS_INCLUDE}")); - assert!( - document.len() > 1_400_000, - "fixture must exceed the real page size, got {}", - document.len() - ); - - let assembled = assemble(&document, FRAGMENT).expect("should assemble"); - let expected = document.replace(ESI_BIDS_INCLUDE, FRAGMENT); - - assert_eq!( - assembled.len(), - expected.len(), - "assembly dropped {} bytes of a {}-byte document", - expected.len() as i64 - assembled.len() as i64, - document.len() - ); - assert!( - assembled.ends_with(""), - "the document must not be cut short; it ends with: {:?}", - &assembled[assembled.len().saturating_sub(80)..] - ); - } - - #[test] - fn dollar_signs_in_the_document_are_not_treated_as_esi_variables() { - // ESI interpolates `$(VAR)`, and a React Server Components payload is full of - // dollar signs: `[\"$\",\"$L1b\",null,…]` is how RSC encodes element references. - // - // A real page lost ~750KB through this parser while the byte-split path served it - // intact, and the two diverged exactly at `self.__next_f.push([1,"15:[\"$\",…`. - // Every fixture here until now was dollar-free. - let payload = r#""#; - let document = format!( - "

before

{payload}

after

{ESI_BIDS_INCLUDE}" - ); - - let assembled = assemble(&document, FRAGMENT).expect("should assemble"); - - assert!( - assembled.contains("after"), - "content after a dollar sign must survive: {assembled}" - ); - assert_eq!( - assembled, - document.replace(ESI_BIDS_INCLUDE, FRAGMENT), - "a document containing `$` must survive byte-identically apart from the seam" - ); - } - - #[test] - fn the_crate_truncates_a_script_larger_than_its_chunk_size() { - // Asserts the *defect*, deliberately. This is why the render path no longer uses - // this crate: `esi` 0.7.1 empties its buffer before parsing and never restores - // those bytes when the parser returns `Incomplete`, so any element larger than - // its 16KB `chunk_size` loses content. Next.js streams its RSC payload as a few - // enormous `self.__next_f.push(...)` scripts, so a real 1.4MB page came back at - // 697KB and the browser showed an error boundary. - // - // Total size is not the trigger — a 1.4MB document of small scripts is fine, and - // that is why every fixture here passed for weeks. The size of one element is. - // Raising `chunk_size` moves the threshold rather than removing it. - // - // If this ever starts failing, the crate has been fixed and edge assembly could - // be reconsidered on its merits rather than ruled out on this one. - let payload = "x".repeat(120_000); - let document = format!( - "

before

\ - \ -

after

{ESI_BIDS_INCLUDE}" - ); - - let assembled = assemble(&document, FRAGMENT).expect("should assemble"); - let expected = document.replace(ESI_BIDS_INCLUDE, FRAGMENT); - - assert!( - assembled.len() < expected.len(), - "the crate is expected to drop bytes here; if it no longer does, this test \ - and the decision it justifies both need revisiting" - ); - } - - #[test] - fn script_bearing_fragments_are_spliced_verbatim() { - // ESI substitutes bytes without escaping, which is exactly why the fragment - // endpoint must return markup rather than JSON. This pins that behaviour, since - // an `esi` release that started escaping would silently turn every fragment - // into visible text. - let fragment = ""; - let assembled = assemble(&template_with_include(), fragment).expect("should assemble"); - - assert!( - assembled.contains(fragment), - "the fragment must be spliced verbatim: {assembled}" - ); - } -} diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 5ffb7226a..603ffdd9b 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -29,7 +29,6 @@ mod app; mod backend; mod compat; mod ec_kv; -mod esi_assembly; mod logging; mod management_api; mod middleware; diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs index d15f8ed55..255fa9d07 100644 --- a/crates/trusted-server-adapter-fastly/src/template_cache.rs +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -238,16 +238,16 @@ mod tests { // the other's template. let cache = cache(); let esi = key("https://example.com/mode-split"); - let mut client_fill = esi.clone(); - client_fill.assembly_mode = AssemblyMode::ClientFill; + let mut inline = esi.clone(); + inline.assembly_mode = AssemblyMode::Inline; let body = b"esi-template".to_vec(); run(cache.put(&esi, &metadata_for(&body), body)).expect("should store"); assert_eq!( - run(cache.get(&client_fill)).err(), + run(cache.get(&inline)).err(), Some(TemplateCacheMiss::NotFound), - "client-fill must not read the ESI arm's template" + "inline must not read the ESI arm's template" ); } diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 14bc26a9e..e47665a65 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -186,15 +186,14 @@ fn derive_section(path: &str, section_root: &str, section_segment: usize) -> Str /// How per-user ad state reaches the page. /// /// `Inline` is the shipped behaviour: the auction result is injected before -/// `` and the root document is therefore uncacheable. The other two serve -/// a request-neutral shared template and fill the per-user holes afterwards — -/// `ClientFill` from the browser, `Esi` at the edge. +/// `` and the root document is therefore uncacheable. `Esi` stores a +/// request-neutral shared template and fills its per-request byte seam at the edge. /// /// Spike-only, for the #1009 ESI validation. Remove with the spike. /// /// # Why the template must be request-neutral /// -/// Under `ClientFill` and `Esi` the template is shared across visitors, so +/// Under `Esi` the template is shared across visitors, so /// nothing whose *presence* depends on the request may appear in it — not merely /// nothing whose *value* does. `tsjs.adSlots` is the trap: its content is derived /// from config and path, but whether it is emitted at all is gated on consent, @@ -207,9 +206,10 @@ pub enum AssemblyMode { /// Inject bids inline before ``. Root uncacheable. Shipped behaviour. #[default] Inline, - /// Serve a shared template; the browser fetches the per-user fragment. - ClientFill, - /// Serve a shared template; assemble the fragment at the edge with ESI. + /// Serve a shared template; assemble its inert marker with an exact byte split. + /// + /// The operator-facing spelling remains `esi` for continuity, but no general + /// purpose ESI parser executes on this path. Esi, } @@ -1959,11 +1959,7 @@ mod tests { #[test] fn assembly_mode_deserializes_each_variant() { - for (raw, expected) in [ - ("inline", AssemblyMode::Inline), - ("client_fill", AssemblyMode::ClientFill), - ("esi", AssemblyMode::Esi), - ] { + for (raw, expected) in [("inline", AssemblyMode::Inline), ("esi", AssemblyMode::Esi)] { let toml = format!( r#" gam_network_id = "99999" @@ -1978,6 +1974,15 @@ mod tests { "should resolve `{raw}` to {expected:?}" ); } + + let removed_mode = r#" + gam_network_id = "99999" + assembly_mode = "client_fill" + "#; + assert!( + toml::from_str::(removed_mode).is_err(), + "client_fill is outside #1009's ESI byte-seam design and must be rejected" + ); } #[test] diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index d10c0252f..559fe0227 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -166,8 +166,7 @@ impl StreamProcessor for HtmlWithPostProcessing { /// §6.7. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub enum BodyCloseInjection { - /// Emit nothing. Either no slots matched under the inline path, or a - /// client-fill mode where the browser fetches the fragment unprompted. + /// Emit nothing because no slots matched under the inline path. #[default] None, /// Read the auction result from `ad_bids_state` and inject it, falling back to diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index b2e1c3446..3dd8f5709 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -36,7 +36,6 @@ mod error; mod http; mod image_optimizer; mod kv; -pub mod template_assembly; mod template_cache; #[cfg(test)] pub(crate) mod test_support; @@ -54,9 +53,6 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; -pub use template_assembly::{ - PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, -}; pub use template_cache::REPLAYABLE_POLICY_HEADERS; pub use template_cache::{ PlatformTemplateCache, TEMPLATE_SCHEMA_VERSION, TemplateCacheError, TemplateCacheKey, diff --git a/crates/trusted-server-core/src/platform/template_assembly.rs b/crates/trusted-server-core/src/platform/template_assembly.rs deleted file mode 100644 index cdf3033d9..000000000 --- a/crates/trusted-server-core/src/platform/template_assembly.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! Edge assembly: turn a shared template plus a per-user fragment into a document. -//! -//! Kept behind a trait for the same reason as the template cache: the only -//! implementation that exists uses a Fastly-only crate, and core must stay portable. -//! Adapters without one get [`UnavailableTemplateAssembler`], which refuses rather than -//! guessing. -//! -//! **Ordering this module exists to protect.** The template is stored *before* assembly -//! and assembled *after* — never the reverse. Storing post-assembly would put one -//! visitor's bids in a cache shared with the next, which is the C3 the design forbids. -//! Splitting store from assemble into two call sites is what makes that ordering -//! visible instead of implicit. -//! -//! Spike-only, for the #1009 ESI validation. - -use core::fmt; - -/// Why assembly could not produce a document. -#[derive(Debug, derive_more::Display)] -pub enum TemplateAssemblyError { - /// The adapter has no assembler. - /// - /// Not a failure to be papered over: reaching here means a shared-template mode is - /// configured on an adapter that cannot serve one, and the honest response is an - /// error rather than a page with an unresolved marker in it. - #[display("this adapter cannot assemble shared templates")] - Unsupported, - /// The assembler ran and failed. - #[display("template assembly failed: {message}")] - Failed { - /// What the underlying assembler reported. - message: String, - }, -} - -impl core::error::Error for TemplateAssemblyError {} - -/// Splices a per-user fragment into a shared template. -pub trait PlatformTemplateAssembler: Send + Sync { - /// Produce the document served to this visitor. - /// - /// # Errors - /// - /// Returns [`TemplateAssemblyError::Unsupported`] when the adapter has no - /// assembler, or [`TemplateAssemblyError::Failed`] when the template could not be - /// processed. - fn assemble(&self, template: &str, fragment: &str) -> Result; -} - -impl fmt::Debug for dyn PlatformTemplateAssembler { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("PlatformTemplateAssembler") - } -} - -/// The default: no assembler. -/// -/// Refuses rather than returning the template unchanged. Returning it unchanged would -/// serve a page whose ad markup is a literal `esi:include` — a page that looks like it -/// worked, renders no ads, and reports no error. -#[derive(Debug, Default, Clone, Copy)] -pub struct UnavailableTemplateAssembler; - -impl PlatformTemplateAssembler for UnavailableTemplateAssembler { - fn assemble(&self, _template: &str, _fragment: &str) -> Result { - Err(TemplateAssemblyError::Unsupported) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn the_null_assembler_refuses_rather_than_passing_the_template_through() { - // Passing it through is the tempting default and the wrong one: the visitor - // gets a page with a raw `esi:include` in it, no ads, and no error anywhere. - let error = UnavailableTemplateAssembler - .assemble( - "", - "", - ) - .expect_err("an adapter with no assembler must refuse"); - - assert!(matches!(error, TemplateAssemblyError::Unsupported)); - } -} diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index 03dfa8b78..6e33cc003 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -488,7 +488,7 @@ mod tests { let base = key().to_cache_key(); let mut mode = key(); - mode.assembly_mode = AssemblyMode::ClientFill; + mode.assembly_mode = AssemblyMode::Inline; assert_ne!( mode.to_cache_key(), base, diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 21fd164d2..a1e48b11c 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -173,11 +173,6 @@ pub struct RuntimeServices { /// per request rather than failing. Spike-only; see /// [`crate::platform::template_cache`]. pub(crate) template_cache: Arc, - /// Edge assembler for shared templates. Defaults to - /// [`UnavailableTemplateAssembler`], which refuses rather than serving a document - /// with an unresolved marker in it. Spike-only; see - /// [`crate::platform::template_assembly`]. - pub(crate) template_assembler: Arc, /// Dynamic backend registration and name prediction. pub(crate) backend: Arc, /// Outbound HTTP client abstraction. @@ -239,12 +234,6 @@ impl RuntimeServices { &*self.template_cache } - /// The edge template assembler. Spike-only. - #[must_use] - pub fn template_assembler(&self) -> &dyn super::PlatformTemplateAssembler { - &*self.template_assembler - } - /// Returns the dynamic backend service. #[must_use] pub fn backend(&self) -> &dyn PlatformBackend { @@ -305,20 +294,6 @@ impl RuntimeServices { ..self } } - - /// Returns a clone of this instance with the template assembler replaced. - /// - /// Spike-only (#1009). - #[must_use] - pub fn with_template_assembler( - self, - assembler: Arc, - ) -> Self { - Self { - template_assembler: assembler, - ..self - } - } } impl fmt::Debug for RuntimeServices { @@ -338,7 +313,6 @@ pub struct RuntimeServicesBuilder { secret_store: Option>, kv_store: Option>, template_cache: Option>, - template_assembler: Option>, backend: Option>, http_client: Option>, geo: Option>, @@ -353,7 +327,6 @@ impl RuntimeServicesBuilder { secret_store: None, kv_store: None, template_cache: None, - template_assembler: None, backend: None, http_client: None, geo: None, @@ -383,16 +356,6 @@ impl RuntimeServicesBuilder { self } - /// Set the edge template assembler. Spike-only. - #[must_use] - pub fn template_assembler( - mut self, - assembler: Arc, - ) -> Self { - self.template_assembler = Some(assembler); - self - } - /// Set the KV store implementation. #[must_use] pub fn kv_store(mut self, kv_store: Arc) -> Self { @@ -460,11 +423,6 @@ impl RuntimeServicesBuilder { template_cache: self .template_cache .unwrap_or_else(|| Arc::new(super::UnavailableTemplateCache)), - // Defaulted to a refusal rather than to a pass-through: an adapter with no - // assembler must not serve a template with an unresolved marker in it. - template_assembler: self - .template_assembler - .unwrap_or_else(|| Arc::new(super::UnavailableTemplateAssembler)), backend: self .backend .expect("should set backend before building RuntimeServices"), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8c37f1743..cad928525 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -980,34 +980,7 @@ pub(crate) fn template_gpt_diagnostics( ) -> Option { match mode { AssemblyMode::Inline => decision, - AssemblyMode::ClientFill | AssemblyMode::Esi => None, - } -} - -/// Whether a root-level auction has any consumer under this assembly mode. -/// -/// Only [`AssemblyMode::Inline`] injects the auction result into the root document. -/// Under the shared-template modes both seams emit nothing, so a dispatched root -/// auction would bill the SSPs, hold the response for the full budget, and have its -/// result discarded with no error and no log. -/// -/// This is the guard for the failure mode described in §5 of -/// `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md`, -/// reached here by an incomplete feature flag rather than by removing the hold. -pub(crate) fn root_auction_is_useful(mode: AssemblyMode) -> bool { - match mode { - AssemblyMode::Inline => true, - // The browser fetches its own bids after load, so a root auction here would be - // a second one nothing reads. - AssemblyMode::ClientFill => false, - // Consumed by edge assembly rather than by a seam. An earlier revision returned - // `false` here on the premise that the fragment would run its own auction via a - // real subrequest. That premise made the arm strictly worse — a self-referencing - // backend, two auction paths, and two auctions per pageview — and it is not what - // `esi` requires: `PendingFragmentContent::CompletedRequest` lets the include be - // satisfied from bytes already in hand. So the auction already in flight *is* - // the fragment, and it is very much consumed. - AssemblyMode::Esi => true, + AssemblyMode::Esi => None, } } @@ -1045,18 +1018,13 @@ fn configured_assembly_mode(settings: &Settings) -> AssemblyMode { /// /// The property that decides whether a template is *expected* to have a hole in it, and /// therefore whether the absence of one is a defect or the design. Only `Esi` splices per -/// reader; `ClientFill` deliberately injects nothing there, because the browser fetches -/// its own bids after load. -/// -/// Treating the marker as universal is what made `ClientFill`'s cache inert: every hit -/// failed the marker check, was logged as transform drift, and fell back to the origin, -/// so that mode stored templates it could never read back. +/// reader. /// /// Matched exhaustively rather than compared against `Esi`, so a new mode has to state /// its answer here instead of silently inheriting one. fn mode_emits_seam_marker(mode: AssemblyMode) -> bool { match mode { - AssemblyMode::Inline | AssemblyMode::ClientFill => false, + AssemblyMode::Inline => false, AssemblyMode::Esi => true, } } @@ -1112,8 +1080,6 @@ pub(crate) fn body_close_injection( BodyCloseInjection::None } } - // The browser fetches the fragment unprompted; nothing to emit. - AssemblyMode::ClientFill => BodyCloseInjection::None, // Constant across every request that reaches the transform — which is what // makes it safe in a shared template. AssemblyMode::Esi => BodyCloseInjection::Marker(SEAM_BIDS_MARKER.to_string()), @@ -1450,16 +1416,6 @@ pub async fn buffer_publisher_response_async( template, mut params, } => { - // Asked of the mode before anything is split, because a mode that emits no - // seam stores a template with no hole in it — and demanding one anyway is - // what turned every `client_fill` hit into a 500 here and into a truncated - // body on the streaming finalizer. The mode is read from settings rather - // than carried: the cache key covers `assembly_mode`, so a template can only - // be read back by the mode that wrote it. - if !mode_emits_seam_marker(configured_assembly_mode(settings)) { - abandon_auction_without_a_seam(services, &mut params).await; - return Ok(serve_seamless_template(response, template)); - } // Buffered adapters have no streaming to preserve, so eager assembly costs // them nothing. The streaming finalizer must not do this. if let Some(dispatched) = params.dispatched_auction.take() { @@ -1579,52 +1535,6 @@ fn assemble_if_shared( Ok(out) } -/// Serves a cached template unchanged, for a mode whose `` seam injects nothing. -/// -/// Under [`AssemblyMode::ClientFill`] the stored template *is* the response: the browser -/// fetches its own bids after load, so there is no per-reader splice and nothing to wait -/// for. Every byte is already in hand, so the length is known and stated — -/// [`build_cached_template_response`] omits it because the assembled length of an `esi` -/// response is not known until bids resolve, which does not apply here. -fn serve_seamless_template( - mut response: Response, - template: Vec, -) -> Response { - response.headers_mut().insert( - http::header::CONTENT_LENGTH, - http::HeaderValue::from(template.len() as u64), - ); - *response.body_mut() = EdgeBody::from(template); - response -} - -/// Reports a dispatched auction that a seamless mode has no way to deliver. -/// -/// Unreachable by construction: [`root_auction_is_useful`] refuses to dispatch under a -/// mode that injects nothing, so there is never anything in flight here. It is spelled -/// out anyway because the alternative is dropping a [`DispatchedAuction`] — real SSP -/// requests, already billed — with no error and no log, which is the silent-waste -/// signature the gate exists to prevent. -async fn abandon_auction_without_a_seam( - services: &RuntimeServices, - params: &mut OwnedProcessResponseParams, -) { - let Some(dispatched) = params.dispatched_auction.take() else { - return; - }; - log::warn!( - "Server-side auction dispatched under an assembly mode with no seam to deliver \ - it into; in-flight SSP bid requests will not be collected" - ); - emit_abandoned_auction( - services, - params.auction_observation.take(), - dispatched, - "seamless_assembly_mode", - ) - .await; -} - /// Fingerprint of everything that changes the injected markup for a given URL. /// /// Two inputs, because either alone under-invalidates: @@ -1691,7 +1601,7 @@ fn integration_fingerprint(settings: &Settings) -> String { /// fell back to inline and therefore carries no marker; validating or splitting it would /// fail and turn an ordinary bypass — the common case against a real origin — into a /// 500. Both conditions, and neither alone: only `Esi` emits a marker, and only an -/// authorized response has one to find. `ClientFill` is authorized but marker-free. +/// authorized response has one to find. fn response_carries_a_seam_marker(was_authorized: bool, settings: &Settings) -> bool { was_authorized && mode_emits_seam_marker(configured_assembly_mode(settings)) } @@ -1981,15 +1891,6 @@ pub async fn publisher_response_into_streaming_response( return Ok(response); } - // The buffered finalizer's counterpart; see the note there. Placed before the - // stream is built rather than inside it: a seamless template has no seam to - // wait at, so there is nothing for a stream to interleave, and failing inside - // the stream could only truncate a response whose headers had already gone. - if !mode_emits_seam_marker(configured_assembly_mode(&settings)) { - abandon_auction_without_a_seam(&services, &mut params).await; - return Ok(serve_seamless_template(response, template)); - } - let services = services.clone(); let settings = Arc::clone(&settings); let orchestrator = Arc::clone(&orchestrator); @@ -3601,16 +3502,6 @@ pub async fn handle_publisher_request( let mut auction_request_for_telemetry: Option = None; let mut dispatched_auction = if matched_slots.is_empty() { None - } else if !root_auction_is_useful(assembly_mode) { - // `ClientFill` injects nothing and the browser fetches its own bids, so - // dispatching here would send real SSP requests, hold the response for the full - // auction budget, and then discard the result with no error and no log — the - // silent-waste signature §5 of the design doc is entirely about. - log::debug!( - "skipping root auction dispatch: assembly mode {assembly_mode:?} has no \ - consumer for the result" - ); - None } else { // Telemetry attribution must use the same publisher identity as the // outbound bid request. On the navigation path `request_host` is the @@ -3870,10 +3761,7 @@ pub async fn handle_publisher_request( // it means the transform changed without the version moving. // // Asked of the mode, not of every template. The key covers - // `assembly_mode`, so a hit was stored by this same mode — and a mode - // that emits no marker stores a template with none. Demanding one - // unconditionally made every `client_fill` hit fail this check, report - // drift that had not happened, and refetch the origin. + // `assembly_mode`, so a hit was stored by this same mode. let seam_check = mode_emits_seam_marker(assembly_mode) .then(|| split_template_at_seam(&entry.body).err()) .flatten(); @@ -4997,7 +4885,7 @@ pub(crate) fn c2_bypass_reason( /// Under [`AssemblyMode::Inline`] the response is per-navigation and not shared, /// so emitting `tsjs.adSlots` only when the ad stack runs is correct. /// -/// Under [`AssemblyMode::ClientFill`] and [`AssemblyMode::Esi`] the document is a +/// Under [`AssemblyMode::Esi`] the document is a /// **shared template**, and `should_run_ad_stack` is request-dependent — it folds /// in consent, bot classification, prefetch status and the auction kill switch. /// Emitting conditionally there would freeze the first-filling request's decision @@ -5018,7 +4906,7 @@ pub(crate) fn template_ad_slots_script( request_path: &str, ) -> Option { match mode { - AssemblyMode::ClientFill | AssemblyMode::Esi => None, + AssemblyMode::Esi => None, AssemblyMode::Inline => { if !should_run_ad_stack { return None; @@ -5188,23 +5076,11 @@ fn page_bids_unknown_format() -> Response { /// client-controlled: strip any query string or fragment and force a leading /// `/` so slot `page_patterns` always match against a canonical path shape. /// How the page-bids endpoint serializes its answer. -/// -/// A format rather than a second endpoint. The two forms carry the same data behind -/// the same cross-site gate, so a new path would have duplicated the gate, the -/// deprecation alias and the `private, no-store` header across four adapter routers -/// for a difference in wrapping. -/// -/// Spike-only, for the #1009 ESI validation. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub(crate) enum PageBidsFormat { /// `application/json`. What the SPA navigation hook consumes. #[default] Json, - /// An executable `"#); + + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(source.as_bytes(), true) + .expect("should process bodyless HTML"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + + assert_eq!( + html.matches(MARKER).count(), + 2, + "one source occurrence plus one transform-owned seam must reach normalization" + ); + assert!( + html.ends_with(MARKER), + "the transform-owned fallback must be unambiguously terminal" + ); + } + #[test] fn response_size_does_not_grow_disproportionately() { // Processing must not expand HTML by more than 1.1× (accounts for the diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index 55b08fd46..19d206f97 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -32,7 +32,8 @@ use crate::creative_opportunities::AssemblyMode; /// | ------- | --------- | /// | 1 | `` seam marker was `` | /// | 2 | Marker is the inert comment [`SEAM_BIDS_MARKER`](crate::publisher::SEAM_BIDS_MARKER); the seam hands slots to `scheduleInitialAdInit` instead of assigning them | -pub const TEMPLATE_SCHEMA_VERSION: u32 = 2; +/// | 3 | Canonical collision-safe key, explicit origin freshness, and complete repeated document-policy metadata | +pub const TEMPLATE_SCHEMA_VERSION: u32 = 3; /// Inputs that select one cached template. /// @@ -55,8 +56,8 @@ pub struct TemplateCacheKey { /// Publisher origin identity, including the outbound Host override. Two virtual /// hosts can share a connection target while producing unrelated documents. pub origin_identity: String, - /// A2 and A3 emit different template bytes. Without this they poison each - /// other's entries. + /// Inline and ESI modes emit different template bytes. Without this they poison + /// each other's entries. pub assembly_mode: AssemblyMode, /// Values of the request headers the **origin** declares it varies on, in the /// order the origin listed them. Not a fixed list: the origin is authoritative, @@ -167,6 +168,14 @@ pub const REPLAYABLE_POLICY_HEADERS: &[&str] = &[ "content-security-policy-report-only", "permissions-policy", "referrer-policy", + "strict-transport-security", + "cross-origin-opener-policy", + "cross-origin-embedder-policy", + "cross-origin-resource-policy", + "origin-agent-cluster", + "reporting-endpoints", + "report-to", + "link", "x-frame-options", "x-content-type-options", "content-language", @@ -175,13 +184,15 @@ pub const REPLAYABLE_POLICY_HEADERS: &[&str] = &[ /// Headers the key covers by construction, whatever the operator configured. /// -/// The shared path offers one canonical encoding set upstream and stores decoded identity -/// bytes, so an origin declaring `Vary: Accept-Encoding` is covered without reader input. Without -/// this, that extremely ordinary declaration — any compressing origin sends it — reads +/// The shared path stores decoded identity bytes and negotiates the reader representation +/// only after assembly, so an origin declaring `Vary: Accept-Encoding` is covered without +/// reader input. This assumes those origin variants differ only by HTTP content coding; +/// operators must leave ESI disabled if an origin changes document semantics instead. +/// Without this carve-out, the ordinary declaration sent by any compressing origin reads /// as an uncovered gap and disqualifies the response, so **C2 would never cache anything -/// against a real origin** unless the operator redundantly listed a header the key -/// already covers. Found by review before it could make the spike measure a hit rate of -/// approximately zero and read that as a result. +/// against a real origin** unless the operator redundantly listed a header the transform +/// already normalizes. Found by review before it could make the spike measure a hit rate +/// of approximately zero and read that as a result. const STRUCTURALLY_COVERED: &[&str] = &["accept-encoding"]; /// Request headers to include in the cache key, and where the list comes from. @@ -214,6 +225,11 @@ pub struct VarySpec { impl VarySpec { /// Build from configured header names. + /// + /// # Panics + /// + /// Panics when a name is not a valid HTTP field name. Runtime configuration is + /// validated with [`Self::try_new`] before this constructor is used. #[must_use] pub fn new(names: impl IntoIterator) -> Self { Self::try_new(names).expect("VarySpec names should be validated at configuration load") @@ -258,11 +274,9 @@ impl VarySpec { self.names .iter() .map(|name| { - let header_name = http::header::HeaderName::from_bytes(name.as_bytes()) - .expect("VarySpec names should already be validated"); - let values = headers.contains_key(&header_name).then(|| { + let values = headers.contains_key(name.as_str()).then(|| { headers - .get_all(&header_name) + .get_all(name.as_str()) .iter() .map(|value| value.as_bytes().to_vec()) .collect() @@ -310,8 +324,8 @@ impl VarySpec { /// safe. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TemplateMetadata { - /// Encoding of the stored bytes. Also in the key; stored so a reader need not - /// re-derive it. + /// Encoding of the stored bytes. C2 writes only `identity`; retaining the field in + /// metadata makes corrupt or stale representations fail validation on read. pub content_encoding: String, /// Content type to rebuild the response with. pub content_type: String, @@ -370,23 +384,58 @@ impl TemplateMetadata { for line in text.lines() { let (key, value) = line.split_once('=')?; match key { - "v" => schema_version = Some(value.parse().ok()?), - "ce" => content_encoding = Some(value.to_string()), + "v" => { + if schema_version.replace(value.parse().ok()?).is_some() { + return None; + } + } + "ce" => { + if content_encoding.replace(value.to_string()).is_some() { + return None; + } + } "h" => { - if let Some((name, header_value)) = value.split_once(':') { - policy_headers.push((name.to_string(), header_value.to_string())); + let (name, header_value) = value.split_once(':')?; + let name = http::header::HeaderName::from_bytes(name.as_bytes()).ok()?; + if !REPLAYABLE_POLICY_HEADERS.contains(&name.as_str()) { + return None; + } + http::HeaderValue::from_bytes(header_value.as_bytes()).ok()?; + policy_headers.push((name.as_str().to_string(), header_value.to_string())); + } + "ct" => { + if content_type.replace(value.to_string()).is_some() { + return None; + } + } + "len" => { + if body_len.replace(value.parse().ok()?).is_some() { + return None; } } - "ct" => content_type = Some(value.to_string()), - "len" => body_len = Some(value.parse().ok()?), _ => return None, } } + let content_encoding = content_encoding?; + // Every template is decoded before insert. Accepting another value here would + // let corrupt metadata label plaintext bytes as gzip on a warm hit. + if content_encoding != "identity" { + return None; + } + let content_type = content_type?; + http::HeaderValue::from_bytes(content_type.as_bytes()).ok()?; + if !content_type + .split(';') + .next() + .is_some_and(|media_type| media_type.trim().eq_ignore_ascii_case("text/html")) + { + return None; + } Some(Self { schema_version: schema_version?, policy_headers, - content_encoding: content_encoding?, - content_type: content_type?, + content_encoding, + content_type, body_len: body_len?, }) } @@ -464,6 +513,10 @@ impl TemplateCacheReservation { } /// Fulfil the reservation with a validated template. + /// + /// # Errors + /// + /// Returns the platform cache error when the reservation cannot be fulfilled. pub fn insert( mut self, metadata: &TemplateMetadata, @@ -472,15 +525,23 @@ impl TemplateCacheReservation { ) -> Result<(), TemplateCacheError> { self.inner .take() - .expect("reservation should be consumed once") + .ok_or_else(|| TemplateCacheError::Backend { + message: "template reservation was already consumed".to_string(), + })? .insert(metadata, body, max_age) } /// Explicitly give up the reservation. Drop performs the same operation as a net. + /// + /// # Errors + /// + /// Returns the platform cache error when the reservation cannot be cancelled. pub fn cancel(mut self) -> Result<(), TemplateCacheError> { self.inner .take() - .expect("reservation should be consumed once") + .ok_or_else(|| TemplateCacheError::Backend { + message: "template reservation was already consumed".to_string(), + })? .cancel() } } @@ -498,6 +559,10 @@ impl Drop for TemplateCacheReservation { /// Adapter-specific ownership token returned by a transactional lookup. pub trait PlatformTemplateCacheReservation: Send { /// Insert and discharge the obligation. + /// + /// # Errors + /// + /// Returns an adapter-specific cache error when the insert fails. fn insert( self: Box, metadata: &TemplateMetadata, @@ -506,6 +571,10 @@ pub trait PlatformTemplateCacheReservation: Send { ) -> Result<(), TemplateCacheError>; /// Cancel and allow a waiting request to take ownership. + /// + /// # Errors + /// + /// Returns an adapter-specific cache error when cancellation fails. fn cancel(self: Box) -> Result<(), TemplateCacheError>; } @@ -576,8 +645,8 @@ pub struct TemplateEntry { /// The null object, used by every adapter without a template cache. /// /// Reporting [`TemplateCacheMiss::Unsupported`] rather than erroring means the -/// shared assembly modes degrade to transforming per request on Cloudflare, Axum and -/// Spin instead of failing — the modes stay portable, only the caching is not. +/// ESI assembly mode degrades to transforming per request on Cloudflare, Axum and Spin +/// instead of failing — the mode stays portable, only the caching is not. pub struct UnavailableTemplateCache; #[async_trait::async_trait(?Send)] @@ -741,7 +810,7 @@ mod tests { #[test] fn rendered_key_is_fixed_size_and_contains_no_request_material() { let rendered = key().to_cache_key(); - assert_eq!(rendered.len(), "ts-c2-v2-".len() + 64); + assert_eq!(rendered.len(), "ts-c2-v3-".len() + 64); for sensitive in ["example.com", "/news/article", "rsc", "abc123"] { assert!( !rendered.contains(sensitive), @@ -931,8 +1000,21 @@ mod tests { #[test] fn metadata_round_trips() { let metadata = TemplateMetadata { - content_encoding: "gzip".to_string(), - policy_headers: Vec::new(), + content_encoding: "identity".to_string(), + policy_headers: vec![ + ( + "content-security-policy".to_string(), + "default-src 'self'".to_string(), + ), + ( + "content-security-policy".to_string(), + "script-src 'self'".to_string(), + ), + ( + "link".to_string(), + "; rel=preload; as=script".to_string(), + ), + ], content_type: "text/html; charset=utf-8".to_string(), schema_version: TEMPLATE_SCHEMA_VERSION, body_len: 42, @@ -949,6 +1031,11 @@ mod tests { &b"v=notanumber\nce=gzip\nct=text/html\nlen=1"[..], &b"v=1\nce=gzip\nct=text/html"[..], &b"v=1\nce=gzip\nct=text/html\nlen=1\nunexpected=1"[..], + &b"v=1\nv=1\nce=identity\nct=text/html\nlen=1"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=cache-control:public"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=not-a-policy:value"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=malformed"[..], + &b"v=1\nce=identity\nct=application/json\nlen=1"[..], &[0xff, 0xfe][..], ] { assert_eq!( @@ -959,6 +1046,25 @@ mod tests { } } + #[test] + fn the_policy_allowlist_covers_document_security_and_delivery_headers() { + for required in [ + "strict-transport-security", + "cross-origin-opener-policy", + "cross-origin-embedder-policy", + "cross-origin-resource-policy", + "origin-agent-cluster", + "reporting-endpoints", + "report-to", + "link", + ] { + assert!( + REPLAYABLE_POLICY_HEADERS.contains(&required), + "warm ESI hits must preserve {required}" + ); + } + } + #[tokio::test] async fn the_null_object_reports_unsupported_rather_than_failing() { // Degrading to per-request transformation keeps the shared modes portable on diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8d1345002..468eec04b 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -21,7 +21,7 @@ use std::borrow::Cow; use std::io::Write; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant, SystemTime}; use brotli::Decompressor; use brotli::enc::BrotliEncoderParams; @@ -63,7 +63,7 @@ use crate::platform::{ GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, VarySpec, }; use crate::price_bucket::{PriceGranularity, price_bucket}; -use crate::response_privacy::CDN_CACHE_HEADERS; +use crate::response_privacy::enforce_private_no_store; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{ @@ -74,6 +74,43 @@ use crate::streaming_replacer::create_url_replacer; const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +const HEADER_X_TS_C2_CACHE: &str = "x-ts-c2-cache"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum C2ResponseState { + Hit, + MissReserved, + MissStored, + MissStoreError, + BypassRequest, + BypassResponse, + Unsupported, + Invalid, + BackendError, +} + +impl C2ResponseState { + const fn as_str(self) -> &'static str { + match self { + Self::Hit => "hit", + Self::MissReserved => "miss-reserved", + Self::MissStored => "miss-stored", + Self::MissStoreError => "miss-store-error", + Self::BypassRequest => "bypass-request", + Self::BypassResponse => "bypass-response", + Self::Unsupported => "unsupported", + Self::Invalid => "invalid", + Self::BackendError => "backend-error", + } + } +} + +fn set_c2_response_state(response: &mut Response, state: C2ResponseState) { + response.headers_mut().insert( + HEADER_X_TS_C2_CACHE, + HeaderValue::from_static(state.as_str()), + ); +} fn body_as_reader( body: EdgeBody, @@ -205,11 +242,16 @@ fn restrict_accept_encoding(req: &mut Request) { // origin responds without compression. Adding encodings here would cause the // origin to compress its response even though the client never asked for it, // and the client would then receive content it cannot decode. + if !req.headers().contains_key(header::ACCEPT_ENCODING) { + return; + } let Some(current) = req .headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned) + .get_all(header::ACCEPT_ENCODING) + .iter() + .map(|value| value.to_str().ok()) + .collect::>>() + .map(|values| values.join(", ")) else { return; }; @@ -277,6 +319,158 @@ fn accept_encoding_qvalue(header_value: &str, target: &str) -> Option { matched_qvalue } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReaderEncodingError { + Malformed, + NoAcceptableEncoding, +} + +fn parse_quality_value(value: &str) -> Option { + let value = value.trim(); + let (whole, fraction) = value + .split_once('.') + .map_or((value, None), |(whole, fraction)| (whole, Some(fraction))); + let fraction_is_valid = fraction.is_none_or(|fraction| { + fraction.len() <= 3 && fraction.bytes().all(|byte| byte.is_ascii_digit()) + }); + if !fraction_is_valid { + return None; + } + match whole { + "0" => value.parse().ok(), + "1" if fraction.is_none_or(|fraction| fraction.bytes().all(|byte| byte == b'0')) => { + Some(1.0) + } + _ => None, + } +} + +fn negotiate_reader_compression( + headers: &edgezero_core::http::HeaderMap, +) -> Result { + if !headers.contains_key(header::ACCEPT_ENCODING) { + return Ok(Compression::None); + } + + let mut qualities = Vec::<(String, f32)>::new(); + for field in headers.get_all(header::ACCEPT_ENCODING) { + let field = field.to_str().map_err(|_| ReaderEncodingError::Malformed)?; + for item in field + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + { + let mut parts = item.split(';'); + let token = parts + .next() + .map(str::trim) + .filter(|token| !token.is_empty()) + .ok_or(ReaderEncodingError::Malformed)? + .to_ascii_lowercase(); + if token != "*" && http::HeaderName::from_bytes(token.as_bytes()).is_err() { + return Err(ReaderEncodingError::Malformed); + } + let mut quality = 1.0; + let mut saw_quality = false; + for parameter in parts { + let (name, value) = parameter + .trim() + .split_once('=') + .ok_or(ReaderEncodingError::Malformed)?; + if !name.trim().eq_ignore_ascii_case("q") || saw_quality { + return Err(ReaderEncodingError::Malformed); + } + quality = parse_quality_value(value).ok_or(ReaderEncodingError::Malformed)?; + saw_quality = true; + } + if qualities.iter().any(|(seen, _)| seen == &token) { + return Err(ReaderEncodingError::Malformed); + } + qualities.push((token, quality)); + } + } + + let explicit = |name: &str| { + qualities + .iter() + .find_map(|(candidate, quality)| (candidate == name).then_some(*quality)) + }; + let wildcard = explicit("*"); + let quality_for = |name: &str| explicit(name).or(wildcard).unwrap_or(0.0); + // Identity is implicitly acceptable at q=1 unless explicitly excluded, or a + // wildcard q=0 excludes every unlisted coding. + let identity_quality = + explicit("identity").unwrap_or_else(|| if wildcard == Some(0.0) { 0.0 } else { 1.0 }); + + let candidates = [ + (Compression::Brotli, quality_for("br")), + (Compression::Gzip, quality_for("gzip")), + (Compression::Deflate, quality_for("deflate")), + (Compression::None, identity_quality), + ]; + let mut selected = None; + for (compression, quality) in candidates { + if quality > 0.0 && selected.is_none_or(|(_, best)| quality > best) { + selected = Some((compression, quality)); + } + } + selected + .map(|(compression, _)| compression) + .ok_or(ReaderEncodingError::NoAcceptableEncoding) +} + +fn set_response_compression(response: &mut Response, compression: Compression) { + let encoding = match compression { + Compression::None => None, + Compression::Gzip => Some("gzip"), + Compression::Deflate => Some("deflate"), + Compression::Brotli => Some("br"), + }; + if let Some(encoding) = encoding { + response + .headers_mut() + .insert(header::CONTENT_ENCODING, HeaderValue::from_static(encoding)); + } else { + response.headers_mut().remove(header::CONTENT_ENCODING); + } + let varies_on_encoding = response + .headers() + .get_all(header::VARY) + .iter() + .any(|value| { + value.to_str().is_ok_and(|value| { + value + .split(',') + .any(|name| name.trim().eq_ignore_ascii_case("accept-encoding")) + }) + }); + if !varies_on_encoding { + response + .headers_mut() + .append(header::VARY, HeaderValue::from_static("Accept-Encoding")); + } + response.headers_mut().remove(header::CONTENT_LENGTH); +} + +fn response_compression(response: &Response) -> Compression { + response + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .map(Compression::from_content_encoding) + .unwrap_or(Compression::None) +} + +fn encode_complete_body( + body: Vec, + compression: Compression, +) -> Result, Report> { + let mut encoder = BodyStreamEncoder::new(compression); + let mut encoded = encoder.encode_chunk(body)?; + encoded.extend_from_slice(&encoder.finish()?); + Ok(encoded) +} + /// Unified tsjs static serving: `/static/tsjs=` /// /// Serves two types of bundles: @@ -433,6 +627,7 @@ fn process_response_streaming( body: EdgeBody, output: &mut W, params: &ProcessResponseParams, + output_compression: Compression, ) -> Result<(), Report> { let is_html = is_html_content_type(params.content_type); let is_rsc_flight = @@ -448,7 +643,7 @@ fn process_response_streaming( let compression = Compression::from_content_encoding(params.content_encoding); let config = PipelineConfig { input_compression: compression, - output_compression: compression, + output_compression, chunk_size: 8192, }; // Bound how much decoded gzip output may sit in the heap at once, using the @@ -514,13 +709,21 @@ async fn process_response_streaming_async( params.content_encoding ); - let compression = Compression::from_content_encoding(¶ms.content_encoding); + let input_compression = Compression::from_content_encoding(¶ms.content_encoding); + // A C2 template is always identity bytes. Decode during the transform instead of + // recompressing and immediately decoding the entire buffered result afterwards. + let output_compression = if params.template_cache_key.is_some() { + Compression::None + } else { + input_compression + }; let mut processor = PublisherBodyProcessor::new(params, settings, integration_registry)?; process_body_chunks_async( body, output, &mut processor, - compression, + input_compression, + output_compression, settings.publisher.max_buffered_body_bytes, ) .await @@ -562,11 +765,12 @@ async fn process_body_chunks_async( body: EdgeBody, writer: &mut W, processor: &mut P, - compression: Compression, + input_compression: Compression, + output_compression: Compression, max_body_bytes: usize, ) -> Result<(), Report> { - let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); - let mut encoder = BodyStreamEncoder::new(compression); + let mut decoder = BodyStreamDecoder::new(input_compression, max_body_bytes); + let mut encoder = BodyStreamEncoder::new(output_compression); let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_body_bytes); while let Some(segments) = @@ -998,7 +1202,7 @@ pub(crate) fn template_gpt_diagnostics( /// Carries no URL. Every byte here is a byte every reader of the shared template /// receives, so nothing request-scoped may appear, and keeping a URL out also removes /// any escaping question at the seam. -pub const SEAM_BIDS_MARKER: &str = ""; +pub const SEAM_BIDS_MARKER: &str = ""; /// The mode the operator asked for, before availability is taken into account. /// @@ -1272,7 +1476,14 @@ pub struct OwnedProcessResponseParams { /// lifetime is only known after the origin proves this representation is shareable. pub(crate) struct AuthorizedTemplateStore { reservation: crate::platform::TemplateCacheReservation, - max_age: Duration, + expires_at: Instant, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TemplateStoreOutcome { + Stored, + Expired, + Error, } /// Buffers a [`PublisherResponse`] into a single [`Response`], collecting the @@ -1363,26 +1574,10 @@ pub async fn buffer_publisher_response_async( services, ) .await?; + // Authorized C2 transforms are emitted as identity by + // `process_response_streaming_async`; inline transforms retain the origin + // coding. This avoids recompressing and immediately decoding a full document. let bytes = output.into_inner(); - // Decode before either storing or splicing. Both are textual operations and - // the transform's output follows the origin's encoding. - // - // Only shared modes pay this. Under `Inline` the bytes go out exactly as the - // encoder produced them, still compressed, which is what the encoder is for. - let bytes = if params.template_cache_key.is_some() { - let decoded = decode_transformed_body( - bytes, - ¶ms.content_encoding, - settings.publisher.max_buffered_body_bytes, - )?; - // The response now carries plaintext, so it must stop claiming otherwise. - response - .headers_mut() - .remove(http::header::CONTENT_ENCODING); - decoded - } else { - bytes - }; // Store first, assemble second — never the reverse. The stored bytes are // shared between visitors; the assembled ones carry this visitor's bids. // Swapping these two lines is the C3 leak. @@ -1390,6 +1585,11 @@ pub async fn buffer_publisher_response_async( // request cannot store twice, which would leave nothing for assembly to gate // on. let was_authorized = params.template_cache_key.is_some(); + let bytes = if response_carries_a_seam_marker(was_authorized, settings) { + normalize_fresh_template_seam(bytes) + } else { + bytes + }; // Validate before the store, not after. // // `assemble_if_shared` does the same split and would reject a malformed @@ -1409,8 +1609,23 @@ pub async fn buffer_publisher_response_async( } })?; } - store_template_if_authorized(services, &mut params, &bytes).await; + let store_outcome = store_template_if_authorized(services, &mut params, &bytes).await; + if was_authorized { + set_c2_response_state( + &mut response, + match store_outcome { + Some(TemplateStoreOutcome::Stored) => C2ResponseState::MissStored, + Some(TemplateStoreOutcome::Expired) => C2ResponseState::BypassResponse, + Some(TemplateStoreOutcome::Error) | None => C2ResponseState::MissStoreError, + }, + ); + } let bytes = assemble_if_shared(was_authorized, settings, ¶ms, bytes)?; + let bytes = if was_authorized { + encode_complete_body(bytes, response_compression(&response))? + } else { + bytes + }; response.headers_mut().insert( http::header::CONTENT_LENGTH, http::HeaderValue::from(bytes.len() as u64), @@ -1456,6 +1671,7 @@ pub async fn buffer_publisher_response_async( assembled.extend_from_slice(head); assembled.extend_from_slice(seam.as_bytes()); assembled.extend_from_slice(tail); + let assembled = encode_complete_body(assembled, response_compression(&response))?; response.headers_mut().insert( http::header::CONTENT_LENGTH, http::HeaderValue::from(assembled.len() as u64), @@ -1470,38 +1686,6 @@ pub async fn buffer_publisher_response_async( } } -/// Decodes transformed body bytes so they can be stored and spliced as text. -/// -/// The pipeline pairs input encoding to output encoding, so a compressed origin yields a -/// compressed transform. Everything the shared-template path does afterwards is textual: -/// finding the seam marker, splitting on it, inserting a script. None of that works on -/// compressed bytes — the marker is not present to find, `from_utf8` fails, and a spliced -/// gzip stream is undecodable in the browser. -/// -/// An earlier attempt forced `Accept-Encoding: identity` on the *origin request* instead. -/// That worked and cost far too much: the origin then sent ~674 KB uncompressed where it -/// would have sent ~100 KB, adding seconds to the fetch. The fetch should stay -/// compressed; only the assembled response needs to be text. -/// -/// # Errors -/// -/// Returns an error if the bytes do not decode, which would mean the encoder and the -/// declared `Content-Encoding` disagree. -fn decode_transformed_body( - bytes: Vec, - content_encoding: &str, - max_decoded_bytes: usize, -) -> Result, Report> { - let compression = Compression::from_content_encoding(content_encoding); - if matches!(compression, Compression::None) { - return Ok(bytes); - } - let mut decoder = BodyStreamDecoder::new(compression, max_decoded_bytes); - let mut out = decoder.decode_chunk(bytes::Bytes::from(bytes))?.to_vec(); - out.extend_from_slice(&decoder.finish()?); - Ok(out) -} - /// Splices this visitor's slots and bids into the seam, if the mode assembles. /// /// Uses the same byte split as the hit path, and deliberately **not** the `esi` crate. @@ -1595,7 +1779,7 @@ fn seam_script_for(params: &OwnedProcessResponseParams) -> String { params .seam_ad_slots .as_deref() - .map(|slots| build_seam_script(slots, ¶ms.ad_bids_state.bids())) + .map(|slots| params.ad_bids_state.build_seam_script(slots)) .unwrap_or_default() } @@ -1654,6 +1838,44 @@ fn split_template_at_seam(template: &[u8]) -> Result<(&[u8], &[u8]), SeamError> Ok((&template[..at], &template[at + marker.len()..])) } +/// Make a freshly transformed ESI template contain one unambiguous seam. +/// +/// `lol_html` emits the marker at ``. HTML fragments and malformed-but-browser- +/// renderable documents may have no body handler, so append a terminal seam rather than +/// converting a valid origin 200 into a TS 500. If any later processing creates an +/// ambiguous result, neutralize every existing marker and mint a new terminal seam. That +/// avoids guessing which occurrence belongs to this transform. +fn normalize_fresh_template_seam(mut template: Vec) -> Vec { + let marker = SEAM_BIDS_MARKER.as_bytes(); + let positions = template + .windows(marker.len()) + .enumerate() + .filter_map(|(at, window)| (window == marker).then_some(at)) + .collect::>(); + + if positions.is_empty() { + log::warn!("c2_template_cache transform emitted no body seam; appending a terminal seam"); + template.extend_from_slice(marker); + return template; + } + + if positions.len() > 1 { + let mut escaped = marker.to_vec(); + // Byte 4 is the first byte inside ``; changing it preserves an + // invisible comment and keeps offsets stable. + escaped[4] = b'x'; + for at in &positions { + template[*at..*at + marker.len()].copy_from_slice(&escaped); + } + template.extend_from_slice(marker); + log::warn!( + "c2_template_cache neutralized {} ambiguous seam markers and appended a terminal seam", + positions.len() + ); + } + template +} + /// Why a template could not be split at its seam. #[derive(Debug, derive_more::Display)] pub(crate) enum SeamError { @@ -1691,17 +1913,12 @@ impl core::error::Error for SeamError {} /// Spike-only, for the #1009 ESI validation. fn build_cached_template_response( entry: &crate::platform::TemplateEntry, + reader_compression: Compression, ) -> Result, Report> { let invalid = |what: &str| TrustedServerError::Proxy { message: format!("cached template has an unusable {what}"), }; let mut response = Response::new(EdgeBody::empty()); - // First, not last: see the note above. The assembled response is per-user even - // though the template it was built from is not. - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); response.headers_mut().insert( header::CONTENT_TYPE, HeaderValue::from_str(&entry.metadata.content_type) @@ -1726,13 +1943,20 @@ fn build_cached_template_response( // is what the allowlist encodes. Without this a hit silently dropped the origin's // Content-Security-Policy and framing protection — a weaker page, served faster. for (name, value) in &entry.metadata.policy_headers { - if let (Ok(name), Ok(value)) = ( - header::HeaderName::from_bytes(name.as_bytes()), - HeaderValue::from_str(value), - ) { - response.headers_mut().insert(name, value); - } + let name = header::HeaderName::from_bytes(name.as_bytes()) + .change_context_lazy(|| invalid("policy header name"))?; + if !crate::platform::REPLAYABLE_POLICY_HEADERS.contains(&name.as_str()) { + return Err(Report::new(invalid("policy header allowlist"))); + } + let value = + HeaderValue::from_str(value).change_context_lazy(|| invalid("policy header value"))?; + response.headers_mut().append(name, value); } + // Last, after replay. Metadata decoding rejects cache-controlling names, and this + // terminal stamp is defense in depth for direct/test entries and future format bugs. + enforce_private_no_store(&mut response); + set_response_compression(&mut response, reader_compression); + set_c2_response_state(&mut response, C2ResponseState::Hit); Ok(response) } @@ -1752,10 +1976,15 @@ async fn store_template_if_authorized( _services: &RuntimeServices, params: &mut OwnedProcessResponseParams, bytes: &[u8], -) { - let Some(store) = params.template_cache_key.take() else { - return; - }; +) -> Option { + let store = params.template_cache_key.take()?; + let max_age = store.expires_at.saturating_duration_since(Instant::now()); + if max_age.is_zero() { + log::debug!( + "c2_template_cache store skipped: origin freshness expired during transformation" + ); + return Some(TemplateStoreOutcome::Expired); + } let metadata = crate::platform::TemplateMetadata { // `identity`, not the origin's encoding. The caller decoded before storing, // because the seam split is textual — so recording the origin's encoding here @@ -1767,10 +1996,7 @@ async fn store_template_if_authorized( schema_version: crate::platform::TEMPLATE_SCHEMA_VERSION, body_len: bytes.len() as u64, }; - match store - .reservation - .insert(&metadata, bytes.to_vec(), store.max_age) - { + match store.reservation.insert(&metadata, bytes.to_vec(), max_age) { Ok(()) => { // Reports whether the seam marker made it into the stored bytes. The marker // is deliberately invisible from the outside — assembly replaces it before @@ -1784,8 +2010,12 @@ async fn store_template_if_authorized( .windows(SEAM_BIDS_MARKER.len()) .any(|w| w == SEAM_BIDS_MARKER.as_bytes()) ); + Some(TemplateStoreOutcome::Stored) + } + Err(err) => { + log::warn!("c2_template_cache store failed: {err}"); + Some(TemplateStoreOutcome::Error) } - Err(err) => log::warn!("c2_template_cache store failed: {err}"), } } @@ -1868,6 +2098,17 @@ pub async fn publisher_response_into_streaming_response( let services = services.clone(); let settings = Arc::clone(&settings); let orchestrator = Arc::clone(&orchestrator); + let compression = response_compression(&response); + // Arm the drop warning before constructing the lazy body. A reader can + // disconnect after receiving the cached article prefix but before the seam + // is polled, just like on the ordinary streaming path. + let dispatched_auction = params.dispatched_auction.take().map(|dispatched| { + let telemetry = AuctionTelemetryCarry { + observation: params.auction_observation.take(), + auction_request: params.auction_request.take(), + }; + (DispatchedAuctionGuard::new(dispatched), telemetry) + }); // This is the whole point of the variant. The template's head goes out // immediately, so the article paints while the auction is still running; the @@ -1876,15 +2117,20 @@ pub async fn publisher_response_into_streaming_response( let stream = async_stream::try_stream! { let (head, tail) = split_template_at_seam(&template) .map_err(|e| std::io::Error::other(e.to_string()))?; - yield bytes::Bytes::copy_from_slice(head); + let mut encoder = BodyStreamEncoder::new(compression); + let encoded_head = encoder + .encode_chunk(head.to_vec()) + .map_err(publisher_stream_error)?; + if !encoded_head.is_empty() { + yield bytes::Bytes::from(encoded_head); + } - if let Some(dispatched) = params.dispatched_auction.take() { + if let Some((mut guard, telemetry)) = dispatched_auction + && let Some(dispatched) = guard.take() + { collect_stream_auction( dispatched, - AuctionTelemetryCarry { - observation: params.auction_observation.take(), - auction_request: params.auction_request.take(), - }, + telemetry, &AuctionCollectDeps { price_granularity: params.price_granularity, ad_bids_state: ¶ms.ad_bids_state, @@ -1898,11 +2144,28 @@ pub async fn publisher_response_into_streaming_response( }, ) .await; + guard.disarm(); } let seam = seam_script_for(¶ms); - yield bytes::Bytes::from(seam); - yield bytes::Bytes::copy_from_slice(tail); + if !seam.is_empty() { + let encoded_seam = encoder + .encode_chunk(seam.into_bytes()) + .map_err(publisher_stream_error)?; + if !encoded_seam.is_empty() { + yield bytes::Bytes::from(encoded_seam); + } + } + let encoded_tail = encoder + .encode_chunk(tail.to_vec()) + .map_err(publisher_stream_error)?; + if !encoded_tail.is_empty() { + yield bytes::Bytes::from(encoded_tail); + } + let trailer = encoder.finish().map_err(publisher_stream_error)?; + if !trailer.is_empty() { + yield bytes::Bytes::from(trailer); + } }; *response.body_mut() = EdgeBody::from_stream::<_, std::io::Error>(stream); Ok(response) @@ -2240,7 +2503,13 @@ pub fn stream_publisher_body( gpt_diagnostics: params.gpt_diagnostics.as_ref(), shared_template_authorized: params.template_cache_key.is_some(), }; - process_response_streaming(body, output, &borrowed) + let input_compression = Compression::from_content_encoding(¶ms.content_encoding); + let output_compression = if params.template_cache_key.is_some() { + Compression::None + } else { + input_compression + }; + process_response_streaming(body, output, &borrowed, output_compression) } /// Stream publisher body with a `( } }; - let compression = Compression::from_content_encoding(¶ms.content_encoding); + let input_compression = Compression::from_content_encoding(¶ms.content_encoding); + let output_compression = if params.template_cache_key.is_some() { + Compression::None + } else { + input_compression + }; stream_html_with_auction_hold( body, output, &mut processor, - compression, + input_compression, + output_compression, AuctionCollectCtx { dispatched, telemetry, @@ -2501,6 +2776,8 @@ pub(crate) struct AdBidsState { script: Arc>>, /// The same bids, structured, for the shared-template seam. bids: Arc>>, + /// Optional per-request diagnostics emitted before either bids-script shape. + debug_prefix: Arc>, } #[cfg(test)] @@ -2536,6 +2813,20 @@ impl AdBidsState { self.bids.lock().expect("should lock bid map").clone() } + /// Build the shared-template seam, retaining the same debug prefix as inline. + fn build_seam_script(&self, slots_json: &str) -> String { + let seam = build_seam_script(slots_json, &self.bids()); + let prefix = self + .debug_prefix + .lock() + .expect("should lock bid debug prefix"); + if prefix.is_empty() { + seam + } else { + format!("{prefix}\n{seam}") + } + } + /// Put `comment` immediately before the rendered bids script. /// /// Debug-only, and deliberately confined to the script: the structured map is what @@ -2556,6 +2847,15 @@ impl AdBidsState { *state = Some(comment.to_string()); } } + let mut prefix = self + .debug_prefix + .lock() + .expect("should lock bid debug prefix"); + if prefix.is_empty() { + *prefix = comment.to_string(); + } else { + *prefix = format!("{comment}\n{prefix}"); + } } } @@ -2798,7 +3098,8 @@ async fn stream_html_with_auction_hold( body: EdgeBody, output: &mut W, processor: &mut P, - compression: Compression, + input_compression: Compression, + output_compression: Compression, ctx: AuctionCollectCtx<'_>, ) -> Result<(), Report> { if body.is_stream() { @@ -2807,7 +3108,8 @@ async fn stream_html_with_auction_hold( body, output, processor, - compression, + input_compression, + output_compression, ctx, max_body_bytes, ) @@ -2818,7 +3120,26 @@ async fn stream_html_with_auction_hold( // enforces, matching the streaming arm above and the no-hold buffered path. let max_body_bytes = ctx.deps.settings.publisher.max_buffered_body_bytes; let body = body_as_reader(body)?; - match compression { + if output_compression == Compression::None { + return match input_compression { + Compression::None => body_close_hold_loop(body, output, processor, ctx).await, + Compression::Gzip => { + let decoder = GzipDecodeReader::new(body, max_body_bytes); + body_close_hold_loop(decoder, output, processor, ctx).await + } + Compression::Deflate => { + let decoder = ZlibDecoder::new(body); + body_close_hold_loop(decoder, output, processor, ctx).await + } + Compression::Brotli => { + let decoder = Decompressor::new(body, STREAM_CHUNK_SIZE); + body_close_hold_loop(decoder, output, processor, ctx).await + } + }; + } + + debug_assert_eq!(input_compression, output_compression); + match input_compression { Compression::None => body_close_hold_loop(body, output, processor, ctx).await, Compression::Gzip => { // `GzipDecodeReader` decodes concatenated gzip members (RFC 1952) @@ -2873,7 +3194,8 @@ async fn body_close_hold_loop_stream( body: EdgeBody, writer: &mut W, processor: &mut P, - compression: Compression, + input_compression: Compression, + output_compression: Compression, ctx: AuctionCollectCtx<'_>, max_body_bytes: usize, ) -> Result<(), Report> { @@ -2882,8 +3204,8 @@ async fn body_close_hold_loop_stream( telemetry, deps: collect_refs, } = ctx; - let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); - let mut encoder = BodyStreamEncoder::new(compression); + let mut decoder = BodyStreamDecoder::new(input_compression, max_body_bytes); + let mut encoder = BodyStreamEncoder::new(output_compression); let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_body_bytes); let mut state = AuctionHoldState::new(DispatchedAuctionGuard::new(dispatched), telemetry); @@ -3626,6 +3948,12 @@ pub async fn handle_publisher_request( .is_some_and(CreativeOpportunitiesConfig::origin_is_cookie_independent); let request_requires_origin = request_bypasses_c2(req.headers()) || gpt_diagnostics.requires_private_no_store(); + let reader_compression = negotiate_reader_compression(req.headers()); + let reader_supports_assembly = reader_compression.is_ok(); + // A failed negotiation bypasses C2 below, so this value is used only on an + // admitted path. Keeping an identity fallback avoids making that relationship + // a panic-prone invariant in the public request handler. + let reader_compression = reader_compression.unwrap_or(Compression::None); if should_run_ad_stack { req.headers_mut().remove(header::IF_NONE_MATCH); @@ -3637,14 +3965,21 @@ pub async fn handle_publisher_request( let method_is_cacheable = req.method() == Method::GET; let request_can_use_shared_template = method_is_cacheable && matches!(assembly_mode, AssemblyMode::Esi) + && !request_host.is_empty() && !request_had_authorization && !cookie_disqualifies - && !request_requires_origin; - - // Only advertise encodings the rewrite pipeline can decode and re-encode. A cold - // reservation switches this to the canonical shared offer after lookup; an - // unsupported/backend-failed cache keeps the reader-compatible offer for inline. - restrict_accept_encoding(&mut req); + && !request_requires_origin + && reader_supports_assembly; + + // Only advertise encodings the rewrite pipeline can decode and re-encode. The + // template is normalized to identity after the origin responds, then encoded for + // this reader after assembly; the origin offer itself remains reader-compatible so + // a response-gate bypass can still fall back to inline losslessly. + if !matches!(assembly_mode, AssemblyMode::Esi) || reader_supports_assembly { + restrict_accept_encoding(&mut req); + } else { + log::debug!("c2_template_cache bypass: reader accepts no representation TS can assemble"); + } // Strip the internal `fastly-ssl` scheme signal before forwarding to the // origin. On the EdgeZero path the entry point re-injects this header from // trusted Fastly TLS metadata so in-process scheme detection works; the @@ -3693,6 +4028,8 @@ pub async fn handle_publisher_request( template_fingerprint: template_fingerprint(settings), schema_version: crate::platform::TEMPLATE_SCHEMA_VERSION, }); + let mut c2_response_state = + matches!(assembly_mode, AssemblyMode::Esi).then_some(C2ResponseState::BypassRequest); *req.uri_mut() = target_uri; req.headers_mut().insert( header::HOST, @@ -3752,6 +4089,7 @@ pub async fn handle_publisher_request( "c2_template_cache could not purge unusable URL variants: {purge_err}" ); } + c2_response_state = Some(C2ResponseState::Invalid); } else { // Deliberately *not* assembled here. The auction is still in flight, // and awaiting it now would hold the first byte until it resolves — @@ -3761,7 +4099,7 @@ pub async fn handle_publisher_request( // // Headers are constructed rather than replayed, so no origin header // can reach a second reader through the cache. - let response = build_cached_template_response(&entry)?; + let response = build_cached_template_response(&entry, reader_compression)?; let mut params = build_template_assembly_params( &entry, settings, @@ -3784,9 +4122,11 @@ pub async fn handle_publisher_request( Ok(crate::platform::TemplateCacheLookup::Reserved(reservation)) => { log::debug!("c2_template_cache cold miss: insert reservation acquired"); template_cache_reservation = Some(reservation); + c2_response_state = Some(C2ResponseState::MissReserved); } Ok(crate::platform::TemplateCacheLookup::Unsupported) => { log::debug!("c2_template_cache bypass: platform has no shared cache"); + c2_response_state = Some(C2ResponseState::Unsupported); } Ok(crate::platform::TemplateCacheLookup::Invalid(miss)) => { log::warn!( @@ -3797,22 +4137,15 @@ pub async fn handle_publisher_request( "c2_template_cache could not purge invalid URL variants: {purge_err}" ); } + c2_response_state = Some(C2ResponseState::Invalid); + } + Err(err) => { + log::warn!("c2_template_cache backend failure: {err}; falling back inline"); + c2_response_state = Some(C2ResponseState::BackendError); } - Err(err) => log::warn!("c2_template_cache backend failure: {err}; falling back inline"), } } - // Every actual shared miss offers the same supported set upstream. The stored - // template is decoded to identity, so reader encoding neither changes nor - // partitions C2. Do this only after acquiring a reservation: adapters without C2 - // fall back inline and must retain the reader-compatible offer above. - if template_cache_reservation.is_some() { - req.headers_mut().insert( - header::ACCEPT_ENCODING, - HeaderValue::from_static("br, gzip, deflate"), - ); - } - // SSP requests are already racing through the platform HTTP client, so // origin TTFB tracks origin latency rather than the auction timeout. // @@ -3884,34 +4217,60 @@ pub async fn handle_publisher_request( .and_then(|h| h.to_str().ok()) .unwrap_or_default() .to_string(); - let template_cache_key = - template_cache_reservation.and_then(|reservation| { - match c2_cache_ttl( - assembly_mode, - request_had_authorization, - cookie_disqualifies, - response.status(), - &gate_content_type, - response.headers(), - &settings - .creative_opportunities - .as_ref() - .map(CreativeOpportunitiesConfig::template_cache_vary) - .unwrap_or_else(|| VarySpec::new([])), - ) { - Err(reason) => { - log::debug!("c2_template_cache bypass: {reason}"); - None - } - Ok(ttl) => { - log::debug!("c2_template_cache eligible for {}s", ttl.as_secs()); - Some(AuthorizedTemplateStore { - reservation, - max_age: ttl, - }) - } + let mut template_cache_key = template_cache_reservation.and_then(|reservation| { + match c2_cache_ttl( + assembly_mode, + request_had_authorization, + cookie_disqualifies, + response.status(), + &gate_content_type, + response.headers(), + &settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::template_cache_vary) + .unwrap_or_else(|| VarySpec::new([])), + ) { + Err(reason) => { + log::debug!("c2_template_cache bypass: {reason}"); + None } - }); + Ok(ttl) => { + log::debug!("c2_template_cache eligible for {}s", ttl.as_secs()); + let Some(expires_at) = Instant::now().checked_add(ttl) else { + log::warn!("c2_template_cache bypass: origin freshness cannot be represented"); + return None; + }; + Some(AuthorizedTemplateStore { + reservation, + expires_at, + }) + } + } + }); + let policy_headers = if template_cache_key.is_some() { + match replayable_policy_headers(response.headers()) { + Ok(headers) => headers, + Err(reason) => { + // The eligibility gate checks this same input immediately above. Keep + // the second read fail-closed in case future code mutates the response + // between authorization and metadata capture. + log::warn!( + "c2_template_cache bypass: policy metadata changed after validation ({reason})" + ); + template_cache_key = None; + Vec::new() + } + } + } else { + Vec::new() + }; + if c2_response_state == Some(C2ResponseState::MissReserved) && template_cache_key.is_none() { + c2_response_state = Some(C2ResponseState::BypassResponse); + } + if let Some(state) = c2_response_state { + set_c2_response_state(&mut response, state); + } // Both seams resolve the mode the same way, from the gate's verdict rather than // from configuration. Deciding them independently is what produced a document with @@ -3963,38 +4322,11 @@ pub async fn handle_publisher_request( // `should_run_ad_stack`: a bot, prefetch, kill-switched or consent-denied request can // assemble an empty-bids document and would otherwise keep the origin's public // caching directives, letting a downstream cache serve it to a later eligible reader. - let policy_headers: Vec<(String, String)> = crate::platform::REPLAYABLE_POLICY_HEADERS - .iter() - .filter_map(|name| { - response - .headers() - .get(*name) - .and_then(|value| value.to_str().ok()) - .map(|value| ((*name).to_string(), value.to_string())) - }) - .collect(); - let assembled_response_must_be_private = template_cache_key.is_some(); if (should_run_ad_stack || assembled_response_must_be_private) && is_html_content_type(origin_content_type) { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); - // Every CDN-targeted cache directive, not just the browser-facing - // `Cache-Control` above: an origin emitting any of these would otherwise - // instruct an intermediary to store a synthesized per-navigation - // document. `Surrogate-Control` and `Fastly-Surrogate-Control` cover - // Fastly; `CDN-Cache-Control` is the standard targeted field (RFC 9213) - // and `Cloudflare-CDN-Cache-Control` is the Cloudflare-specific field - // that overrides it there, so both are needed to close the gap on the - // Cloudflare adapter. - for directive in CDN_CACHE_HEADERS { - response.headers_mut().remove(*directive); - } + enforce_private_no_store(&mut response); } let content_type = response @@ -4011,8 +4343,12 @@ pub async fn handle_publisher_request( .get(header::CONTENT_ENCODING) .map(|h| h.to_str().unwrap_or_default()) .unwrap_or_default() - .to_lowercase(); + .trim() + .to_ascii_lowercase(); let route = classify_response_route(status, &content_type, &content_encoding, request_host); + if template_cache_key.is_some() { + set_response_compression(&mut response, reader_compression); + } match route { ResponseRoute::PassThrough => { @@ -4733,10 +5069,6 @@ fn match_renderable_slots( .collect() } -/// Build the `tsjs.adSlots` ` ``` -No `esi:include`. One origin fetch for two requests. `private, no-store` on the hit. +No executable ESI tag. One origin fetch for two requests. `private, no-store` on the hit. Cached template 353 bytes against 467 served, so the cache holds the pre-assembly template. All three fragment formats behave: script, JSON, and `400` on a typo. `Inline` unaffected — two fetches for two requests, no C2 activity, no markers. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md index 3fd26124a..5b9a455e7 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -1,11 +1,12 @@ # #1009 ESI Validation Spike -> **Superseded, 2026-08-12.** This document records the investigation, including the -> parser/subrequest and client-fill arms that were later removed. The accepted implementation -> keeps the public `esi` spelling but uses Fastly C2 plus exact byte-seam assembly. See +> **HISTORICAL SPIKE — DO NOT IMPLEMENT.** This document records the investigation, +> including executable ESI tags, parser/subrequests, and a client-fill arm that were all +> removed. Every unchecked item below is historical, not remaining work. The accepted +> implementation keeps the public `esi` spelling but uses Fastly C2 plus exact byte-seam +> assembly. See > [the merge-hardening design](../specs/2026-08-12-1009-esi-merge-hardening-design.md) and -> [implementation plan](./2026-08-12-1009-esi-merge-hardening.md). Unchecked items below are -> historical experiments, not remaining work for the merged design. +> [implementation plan](./2026-08-12-1009-esi-merge-hardening.md). > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development > (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps @@ -357,8 +358,8 @@ consenting first fill serves ad markup to a user who refused. | GPT diagnostics bootstrap | **Fragment** — gated on a per-request cookie/query | Emit **one** unconditional marker at the body-close seam, identical on every request that -reaches the transform. Under `Esi` it is an ``; under `ClientFill` it is -nothing at all, with the client fetching unprompted. +reaches the transform. Under `Esi` it is an executable ESI include tag; under +`ClientFill` it is nothing at all, with the client fetching unprompted. - [ ] **Step 3: Bypass C2 for anything that must not be shared** @@ -696,7 +697,7 @@ for the silent-empty-bids trap, which applies in full. - [x] **Step 0: the mechanism works — DONE.** `9539061e`, hardened in `0597f54e`. Verified under Viceroy with the real `esi` 0.7 crate rather than argued from docs: a -template carrying the `` seam's own `esi:include` comes back with the fragment +template carrying the `` seam's own ESI include tag comes back with the fragment spliced in its place and no unresolved tag left. **The async/sync obstacle is dissolved, not worked around.** `esi`'s fragment dispatcher @@ -718,7 +719,7 @@ ESI), `max_include_depth = 1`, and rendered caching / `edge_control` off because publisher path owns those headers. Nine tests. Four assert the configuration; the rest assert behaviour, including that a -fragment containing its own `esi:include` is spliced as text rather than dispatched, so +fragment containing its own nested ESI include is spliced as text rather than dispatched, so auction data cannot drive fragment requests. **What remains is the call site**, below. Emitting the include and resolving it are both @@ -771,8 +772,8 @@ host and panics on a hostless URL — never use it. Rationale in the spec's §2: bid payloads carry partner-controlled creative markup, so a recursive parse would let an SSP make the edge fetch an arbitrary URL. **Add a unit test -that feeds `` through a creative payload and -asserts no fetch is attempted.** +that feeds a partner-controlled ESI include targeting `http://attacker.example/` through +a creative payload and asserts no fetch is attempted.** - [ ] **Step 3: The fragment must be a script, not the JSON endpoint** @@ -805,11 +806,12 @@ Three more things the naïve marker gets wrong: An exact-path allowlist alone permits `https://attacker.example/_ts/page-bids`. Validate **scheme, authority, method, path, and query** — or better, ignore the marker's URL -entirely and dispatch to a fixed internal backend, treating the `esi:include` as a signal +entirely and dispatch to a fixed internal backend, treating the ESI include as a signal rather than an address. -Add a test that feeds `` -through a creative payload and asserts no outbound fetch is attempted. +Add a test that feeds an ESI include targeting +`https://attacker.example/_ts/page-bids` through a creative payload and asserts no +outbound fetch is attempted. - [ ] **Step 5: Deterministic synthetic fragment first** @@ -857,7 +859,7 @@ Not a phase. Every one of these is a hard fail, independent of any performance r check. - [ ] **Request collapsing** works: concurrent cold requests transform once. - [x] **DCA disabled** DONE — `0597f54e`. Config asserted _and_ behaviour: a fragment - carrying its own `esi:include` is spliced as text rather than dispatched. + carrying its own nested ESI include is spliced as text rather than dispatched. - [ ] **Request collapsing** — not tested, and not testable here. Viceroy is single-threaded, so the concurrent cold-request case cannot be produced. The racing diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md index a4f617554..031ecc50c 100644 --- a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -5,10 +5,11 @@ cross-reference point at it. The subject moved, the path did not._ **Issue:** IABTechLab/trusted-server#1009 · **Date:** 2026-08-08 · -> **Superseded for implementation, 2026-08-12.** This remains the measurement and -> feasibility history. The final branch removes the general ESI parser and client-fill arm, -> retaining `assembly_mode = "esi"` as the operator spelling for Fastly C2 plus exact -> byte-seam assembly. See +> **HISTORICAL RECORD — NOT THE CURRENT IMPLEMENTATION.** This document preserves the +> measurement and feasibility investigation. Every executable ESI tag, parser, and +> subrequest described below belongs to a rejected spike; do not use those sections to +> infer current runtime behavior. The final branch retains `assembly_mode = "esi"` only as +> the operator spelling for Fastly C2 plus exact byte-seam assembly. See > [the merge-hardening design](./2026-08-12-1009-esi-merge-hardening-design.md). **Revised:** 2026-08-10 @@ -34,8 +35,8 @@ cross-reference point at it. The subject moved, the path did not._ > read-through cache (C1) that Stage 0 turns on — purging that needs origin-supplied > keys or the HTTP cache's own surrogate-key surface. > - **The original pipeline ordering was backwards.** It said "order esi → lol*html, -> never the reverse." `lol_html` \_emits* the `esi:include` tags, so ESI must run after -> it. Correct order is in [§6.6](#66-the-esi-pipeline-corrected). +> never the reverse." `lol_html` \_emits* the ESI include tags, so ESI must run after it. +> Correct order is in [§6.6](#66-the-esi-pipeline-corrected). > > The error was inspecting what this repository does and reporting it as what the > platform permits — the same mistake this document criticises #1009 for making in the @@ -103,9 +104,9 @@ detail than the work it recommends. ## 2. Why — the three findings -**ESI is buildable on the pinned SDK, and unvalidated.** `lol_html` emits -`esi:include` tags into a shared template; `fastly::cache::core` stores that template; -the `esi` crate assembles per request on the way out. Everything that requires is +**ESI is buildable on the pinned SDK, and unvalidated.** `lol_html` emits executable ESI +include tags into a shared template; `fastly::cache::core` stores that template; the +`esi` crate assembles per request on the way out. Everything that requires is already a dependency. The real open questions are empirical, not architectural: does it beat a plain client fetch by enough to justify a Fastly-only rendering path, and can per-user leakage be excluded under cold MISS, warm HIT, stale revalidation, and fragment @@ -116,7 +117,7 @@ Two constraints stay true regardless. ESI is **Fastly-only at every API level**, is a per-platform accelerator rather than the architecture, and its maintenance cost belongs in the decision. And its Dynamic Content Assembly must be **explicitly disabled** — bid payloads carry partner-controlled creative markup, so under `DcaMode::Esi` an SSP -could embed `` and make the edge fetch an arbitrary URL. Details in +could embed an ESI include targeting an arbitrary URL and make the edge fetch it. Details in [Appendix E](#appendix-e--esi-notes-condensed). **The auction is already out of band; the hold is ~free.** It is dispatched _before_ @@ -444,7 +445,7 @@ for it. ### 6.6 The ESI pipeline, corrected An earlier revision of this document said "order esi → lol*html, never the reverse." -That is backwards. `lol_html` is what \_emits* the `esi:include` tags; ESI cannot process +That is backwards. `lol_html` is what \_emits* the ESI include tags; ESI cannot process tags that do not exist yet. The correct order: ``` diff --git a/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md index 3707edab7..1e030f159 100644 --- a/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md +++ b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md @@ -190,9 +190,9 @@ capturing it means plumbing the writer position into a `lol_html` end-tag handle does not survive re-encoding. A `find` over a ~100 KB buffered template is free by comparison. -**Why a comment rather than `esi:include`.** An HTML comment is inert. If assembly ever -fails to substitute, the reader sees nothing; an unresolved `esi:include` renders as -visible text. Failure degrades to "no ads" instead of "broken page". +**Why a comment rather than executable ESI markup.** An HTML comment is inert. If +assembly ever fails to substitute, the reader sees nothing; an unresolved ESI include +tag renders as visible text. Failure degrades to "no ads" instead of "broken page". **Why not re-run `lol_html` over the cached template.** It would inject a second tsjs `"; + let assembled = services + .template_assembler() + .assemble(template.as_bytes(), fragment) + .expect("Fastly services should provide ESI assembly"); + + assert_eq!( + assembled, + template + .replace( + trusted_server_core::publisher::AD_ASSEMBLY_SEAM, + std::str::from_utf8(fragment).expect("fragment should be UTF-8") + ) + .into_bytes() + ); + } + /// Builds a router whose `AppState` uses a registry containing the given /// request filters (and no routes), so dispatch-level request-filter /// behavior can be exercised without a real integration. diff --git a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs new file mode 100644 index 000000000..4a92c6f78 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs @@ -0,0 +1,287 @@ +//! Fastly cold-response assembly backed by the repaired `stackpop/esi` parser. +//! +//! C2 stores an inert marker. This module creates one synthetic ESI include only in a +//! request-private working copy, resolves it from an already-built fragment, and never +//! performs an HTTP request. + +use std::io::Cursor; + +use esi::{CacheConfig, Configuration, DcaMode, PendingFragmentContent, Processor}; +use fastly::http::StatusCode; +use fastly::{Request, Response}; +use trusted_server_core::platform::{PlatformTemplateAssembler, TemplateAssemblyError}; +use trusted_server_core::publisher::AD_ASSEMBLY_SEAM; + +const INTERNAL_FRAGMENT_PATH: &str = "/_ts/internal/reader-ad-state"; +const SYNTHETIC_ESI_INCLUDE: &[u8] = b""; + +/// Why the Fastly ESI adapter refused or failed to assemble a document. +#[derive(Debug, derive_more::Display)] +enum EsiAssemblyError { + /// The inert seam marker was missing or repeated. + #[display("expected exactly one inert seam marker, found {count}")] + InvalidMarkerCount { count: usize }, + /// Publisher bytes contained ESI instructions outside TS's synthetic seam. + #[display("publisher-authored ESI directives are not allowed")] + PublisherEsiDirective, + /// The parser dispatched a URL other than TS's one synthetic fragment. + #[display("unexpected fragment request path `{path}` (query present: {has_query})")] + UnexpectedFragmentRequest { path: String, has_query: bool }, + /// The pinned parser could not process the document. + #[display("ESI processing failed: {message}")] + Processing { message: String }, + /// The parser changed bytes outside the one synthetic include. + #[display("ESI output was not an exact seam substitution")] + OutputMismatch, +} + +impl core::error::Error for EsiAssemblyError {} + +/// ESI configuration with every cache- and recursion-sensitive option explicit. +fn assembly_configuration() -> Configuration { + Configuration::default() + .with_escaped(false) + .with_default_dca(DcaMode::None) + .with_inherit_parent_dca(false) + .with_max_include_depth(1) + .with_edge_control(false) + .with_caching(CacheConfig { + is_includes_cacheable: false, + includes_default_ttl: None, + includes_force_ttl: None, + is_rendered_cacheable: false, + rendered_cache_control: false, + rendered_ttl: None, + }) +} + +fn contains_esi_directive(bytes: &[u8]) -> bool { + bytes + .windows(b" Result<(Vec, usize), EsiAssemblyError> { + let marker = AD_ASSEMBLY_SEAM.as_bytes(); + let positions = template + .windows(marker.len()) + .enumerate() + .filter_map(|(at, window)| (window == marker).then_some(at)) + .collect::>(); + if positions.len() != 1 { + return Err(EsiAssemblyError::InvalidMarkerCount { + count: positions.len(), + }); + } + if contains_esi_directive(template) { + return Err(EsiAssemblyError::PublisherEsiDirective); + } + + let at = positions[0]; + let mut working = + Vec::with_capacity(template.len() - marker.len() + SYNTHETIC_ESI_INCLUDE.len()); + working.extend_from_slice(&template[..at]); + working.extend_from_slice(SYNTHETIC_ESI_INCLUDE); + working.extend_from_slice(&template[at + marker.len()..]); + Ok((working, at)) +} + +fn completed_fragment_response( + request: &Request, + fragment: &[u8], +) -> Result { + let path = request.get_path().to_string(); + let has_query = request.get_url().query().is_some(); + if path != INTERNAL_FRAGMENT_PATH || has_query { + return Err(EsiAssemblyError::UnexpectedFragmentRequest { path, has_query }); + } + + Ok(PendingFragmentContent::CompletedRequest(Box::new( + Response::from_status(StatusCode::OK) + .with_header( + fastly::http::header::CONTENT_TYPE, + "text/html; charset=utf-8", + ) + .with_body(fragment.to_vec()), + ))) +} + +fn assemble_with_observer( + template: &[u8], + fragment: &[u8], + on_dispatch: F, +) -> Result, EsiAssemblyError> +where + F: Fn() + 'static, +{ + let (working, seam_at) = template_with_synthetic_include(template)?; + let fragment_len = fragment.len(); + let fragment_response = fragment.to_vec(); + let dispatcher = move |request, _index| { + on_dispatch(); + completed_fragment_response(&request, &fragment_response) + .map_err(|error| esi::ESIError::FragmentRequestError(error.to_string())) + }; + let mut processor = Processor::new(None, assembly_configuration()); + let mut output = Vec::with_capacity(template.len() + fragment_len); + processor + .process_stream(Cursor::new(working), &mut output, Some(&dispatcher), None) + .map_err(|error| EsiAssemblyError::Processing { + message: error.to_string(), + })?; + let expected_len = template.len() - AD_ASSEMBLY_SEAM.len() + fragment_len; + let output_tail_at = seam_at + fragment_len; + let template_tail_at = seam_at + AD_ASSEMBLY_SEAM.len(); + if output.len() != expected_len + || output[..seam_at] != template[..seam_at] + || &output[seam_at..output_tail_at] != fragment + || output[output_tail_at..] != template[template_tail_at..] + { + return Err(EsiAssemblyError::OutputMismatch); + } + Ok(output) +} + +fn assemble(template: &[u8], fragment: &[u8]) -> Result, EsiAssemblyError> { + assemble_with_observer(template, fragment, || {}) +} + +/// Fastly implementation of the core cold-response assembly boundary. +pub struct FastlyTemplateAssembler; + +impl PlatformTemplateAssembler for FastlyTemplateAssembler { + fn assemble(&self, template: &[u8], fragment: &[u8]) -> Result, TemplateAssemblyError> { + assemble(template, fragment).map_err(|error| TemplateAssemblyError::Failed { + message: error.to_string(), + }) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + use trusted_server_core::publisher::AD_ASSEMBLY_SEAM; + + const FRAGMENT: &[u8] = b""; + + fn template(body: &str) -> Vec { + format!("{body}{AD_ASSEMBLY_SEAM}").into_bytes() + } + + #[test] + fn a_script_larger_than_the_parser_chunk_survives_exactly() { + let script = format!( + "", + "x".repeat(120_000) + ); + let document = template(&script); + let dispatches = Arc::new(AtomicUsize::new(0)); + let observed_dispatches = Arc::clone(&dispatches); + + let assembled = assemble_with_observer(&document, FRAGMENT, move || { + observed_dispatches.fetch_add(1, Ordering::Relaxed); + }) + .expect("should assemble a document with a large script"); + + let seam_at = document + .windows(AD_ASSEMBLY_SEAM.len()) + .position(|window| window == AD_ASSEMBLY_SEAM.as_bytes()) + .expect("should find seam"); + let mut expected = Vec::new(); + expected.extend_from_slice(&document[..seam_at]); + expected.extend_from_slice(FRAGMENT); + expected.extend_from_slice(&document[seam_at + AD_ASSEMBLY_SEAM.len()..]); + + assert_eq!( + assembled, expected, + "ESI must alter only the synthetic seam" + ); + assert_eq!(dispatches.load(Ordering::Relaxed), 1); + } + + #[test] + fn missing_and_repeated_markers_are_rejected_before_parsing() { + let missing = assemble(b"plain", FRAGMENT) + .expect_err("should reject a missing marker"); + let repeated = assemble( + format!("{AD_ASSEMBLY_SEAM}{AD_ASSEMBLY_SEAM}").as_bytes(), + FRAGMENT, + ) + .expect_err("should reject repeated markers"); + + assert!(matches!( + missing, + EsiAssemblyError::InvalidMarkerCount { count: 0 } + )); + assert!(matches!( + repeated, + EsiAssemblyError::InvalidMarkerCount { count: 2 } + )); + } + + #[test] + fn every_publisher_esi_directive_form_is_rejected_case_insensitively() { + for directive in [ + "", + "secret", + "x", + "$(HTTP_HOST)", + "text", + "", + ] { + let error = assemble(&template(directive), FRAGMENT) + .expect_err("should reject publisher-authored ESI"); + + assert!(matches!(error, EsiAssemblyError::PublisherEsiDirective)); + } + } + + #[test] + fn fragment_esi_is_emitted_verbatim_and_never_reparsed() { + let fragment = b""; + + let assembled = assemble(&template("article"), fragment).expect("should assemble"); + + assert!( + assembled + .windows(fragment.len()) + .any(|window| window == fragment), + "fragment bytes must remain data" + ); + } + + #[test] + fn dispatcher_rejects_every_url_except_the_synthetic_internal_one() { + let unexpected = fastly::Request::get("https://example.com/not-the-seam"); + let with_query = + fastly::Request::get("https://example.com/_ts/internal/reader-ad-state?publisher=1"); + + assert!(matches!( + completed_fragment_response(&unexpected, FRAGMENT), + Err(EsiAssemblyError::UnexpectedFragmentRequest { .. }) + )); + assert!(matches!( + completed_fragment_response(&with_query, FRAGMENT), + Err(EsiAssemblyError::UnexpectedFragmentRequest { .. }) + )); + } + + #[test] + fn configuration_cannot_cache_or_reparse_reader_state() { + let configuration = assembly_configuration(); + + assert!(!configuration.cache.is_includes_cacheable); + assert!(configuration.cache.includes_default_ttl.is_none()); + assert!(configuration.cache.includes_force_ttl.is_none()); + assert!(!configuration.cache.is_rendered_cacheable); + assert!(!configuration.cache.rendered_cache_control); + assert!(configuration.cache.rendered_ttl.is_none()); + assert_eq!(configuration.default_dca, DcaMode::None); + assert!(!configuration.inherit_parent_dca); + assert_eq!(configuration.max_include_depth, 1); + assert!(!configuration.enable_edge_control); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index ed5186f69..07c042ea8 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -29,6 +29,7 @@ mod app; mod backend; mod compat; mod ec_kv; +mod esi_assembly; mod logging; mod management_api; mod middleware; diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 02fe0c65b..7c80f9e12 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -12,6 +12,7 @@ //! - [`PlatformBackend`] — dynamic backend registration //! - [`PlatformHttpClient`] — outbound HTTP client //! - [`PlatformGeo`] — geographic information lookup +//! - [`PlatformTemplateAssembler`] — cold-response shared-template assembly //! //! ## Platform-Agnostic Components //! @@ -36,6 +37,7 @@ mod error; mod http; mod image_optimizer; mod kv; +mod template_assembly; mod template_cache; #[cfg(test)] pub(crate) mod test_support; @@ -53,6 +55,9 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; +pub use template_assembly::{ + PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, +}; pub use template_cache::REPLAYABLE_POLICY_HEADERS; pub use template_cache::{ PlatformTemplateCache, PlatformTemplateCacheReservation, TEMPLATE_SCHEMA_VERSION, diff --git a/crates/trusted-server-core/src/platform/template_assembly.rs b/crates/trusted-server-core/src/platform/template_assembly.rs new file mode 100644 index 000000000..441e7ca31 --- /dev/null +++ b/crates/trusted-server-core/src/platform/template_assembly.rs @@ -0,0 +1,77 @@ +//! Platform boundary for assembling a shared template with reader-specific state. +//! +//! Core owns the cache-safety ordering and the portable byte-seam fallback. An adapter +//! may provide a richer assembler for the cold response after the reader-neutral +//! template has been stored. + +use core::fmt; + +/// Why a platform assembler could not produce a document. +#[derive(Debug, derive_more::Display)] +pub enum TemplateAssemblyError { + /// The adapter has no template assembler. + #[display("this adapter cannot assemble shared templates")] + Unsupported, + /// The platform assembler rejected or could not process the document. + #[display("template assembly failed: {message}")] + Failed { + /// Human-readable failure context. + message: String, + }, +} + +impl core::error::Error for TemplateAssemblyError {} + +/// Assembles reader-specific state into a shared HTML template. +pub trait PlatformTemplateAssembler: Send + Sync { + /// Produce the complete document served to this reader. + /// + /// # Errors + /// + /// Returns [`TemplateAssemblyError`] when the adapter cannot assemble the template. + fn assemble(&self, template: &[u8], fragment: &[u8]) -> Result, TemplateAssemblyError>; +} + +impl fmt::Debug for dyn PlatformTemplateAssembler { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformTemplateAssembler") + } +} + +/// Default assembler used by adapters that do not provide platform assembly. +#[derive(Debug, Default, Clone, Copy)] +pub struct UnavailableTemplateAssembler; + +impl PlatformTemplateAssembler for UnavailableTemplateAssembler { + fn assemble( + &self, + _template: &[u8], + _fragment: &[u8], + ) -> Result, TemplateAssemblyError> { + Err(TemplateAssemblyError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unavailable_assembler_refuses_the_document() { + let error = UnavailableTemplateAssembler + .assemble(b"", b"") + .expect_err("should refuse when platform assembly is unavailable"); + + assert!(matches!(error, TemplateAssemblyError::Unsupported)); + } + + #[test] + fn assembler_contract_is_object_safe() { + let assembler: Box = Box::new(UnavailableTemplateAssembler); + + assert!(matches!( + assembler.assemble(b"template", b"fragment"), + Err(TemplateAssemblyError::Unsupported) + )); + } +} diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index a1e48b11c..e9e02e524 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -173,6 +173,11 @@ pub struct RuntimeServices { /// per request rather than failing. Spike-only; see /// [`crate::platform::template_cache`]. pub(crate) template_cache: Arc, + /// Platform-specific cold-response template assembler. + /// + /// Defaults to [`super::UnavailableTemplateAssembler`]. Core retains a portable + /// byte-seam fallback when this service is unavailable or rejects a document. + pub(crate) template_assembler: Arc, /// Dynamic backend registration and name prediction. pub(crate) backend: Arc, /// Outbound HTTP client abstraction. @@ -234,6 +239,12 @@ impl RuntimeServices { &*self.template_cache } + /// Returns the platform-specific cold-response template assembler. + #[must_use] + pub fn template_assembler(&self) -> &dyn super::PlatformTemplateAssembler { + &*self.template_assembler + } + /// Returns the dynamic backend service. #[must_use] pub fn backend(&self) -> &dyn PlatformBackend { @@ -294,6 +305,18 @@ impl RuntimeServices { ..self } } + + /// Returns a clone of this instance with the template assembler replaced. + #[must_use] + pub fn with_template_assembler( + self, + assembler: Arc, + ) -> Self { + Self { + template_assembler: assembler, + ..self + } + } } impl fmt::Debug for RuntimeServices { @@ -313,6 +336,7 @@ pub struct RuntimeServicesBuilder { secret_store: Option>, kv_store: Option>, template_cache: Option>, + template_assembler: Option>, backend: Option>, http_client: Option>, geo: Option>, @@ -327,6 +351,7 @@ impl RuntimeServicesBuilder { secret_store: None, kv_store: None, template_cache: None, + template_assembler: None, backend: None, http_client: None, geo: None, @@ -356,6 +381,16 @@ impl RuntimeServicesBuilder { self } + /// Set the platform-specific cold-response template assembler. + #[must_use] + pub fn template_assembler( + mut self, + assembler: Arc, + ) -> Self { + self.template_assembler = Some(assembler); + self + } + /// Set the KV store implementation. #[must_use] pub fn kv_store(mut self, kv_store: Arc) -> Self { @@ -423,6 +458,9 @@ impl RuntimeServicesBuilder { template_cache: self .template_cache .unwrap_or_else(|| Arc::new(super::UnavailableTemplateCache)), + template_assembler: self + .template_assembler + .unwrap_or_else(|| Arc::new(super::UnavailableTemplateAssembler)), backend: self .backend .expect("should set backend before building RuntimeServices"), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 29f8802e9..ee0fb7816 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -75,6 +75,7 @@ use crate::streaming_replacer::create_url_replacer; const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); const HEADER_X_TS_C2_CACHE: &str = "x-ts-c2-cache"; +const HEADER_X_TS_ASSEMBLY: &str = "x-ts-assembly"; #[derive(Clone, Copy, PartialEq, Eq)] enum C2ResponseState { @@ -112,6 +113,30 @@ fn set_c2_response_state(response: &mut Response, state: C2ResponseSta ); } +#[derive(Clone, Copy, PartialEq, Eq)] +enum AssemblyResponseState { + EsiParser, + ByteSeamFallback, + ByteSeam, +} + +impl AssemblyResponseState { + const fn as_str(self) -> &'static str { + match self { + Self::EsiParser => "esi-parser", + Self::ByteSeamFallback => "byte-seam-fallback", + Self::ByteSeam => "byte-seam", + } + } +} + +fn set_assembly_response_state(response: &mut Response, state: AssemblyResponseState) { + response.headers_mut().insert( + HEADER_X_TS_ASSEMBLY, + HeaderValue::from_static(state.as_str()), + ); +} + fn body_as_reader( body: EdgeBody, ) -> Result, Report> { @@ -1610,18 +1635,44 @@ pub async fn buffer_publisher_response_async( } })?; } + // Publisher-authored ESI is outside TS's template schema. If it entered C2, + // a warm hit would stream it without passing through the cold parser that + // rejects it. Revoke the reservation before storage and keep this response + // on the portable byte-seam path. + let contains_publisher_esi = was_authorized && contains_esi_directive(&bytes); + if contains_publisher_esi { + log::warn!( + "c2_template_cache bypass: transformed response contains publisher-authored ESI" + ); + params.template_cache_key.take(); + } let store_outcome = store_template_if_authorized(services, &mut params, &bytes).await; if was_authorized { set_c2_response_state( &mut response, - match store_outcome { - Some(TemplateStoreOutcome::Stored) => C2ResponseState::MissStored, - Some(TemplateStoreOutcome::Expired) => C2ResponseState::BypassResponse, - Some(TemplateStoreOutcome::Error) | None => C2ResponseState::MissStoreError, + match (contains_publisher_esi, store_outcome) { + (true, _) => C2ResponseState::BypassResponse, + (false, Some(TemplateStoreOutcome::Stored)) => C2ResponseState::MissStored, + (false, Some(TemplateStoreOutcome::Expired)) => { + C2ResponseState::BypassResponse + } + (false, Some(TemplateStoreOutcome::Error) | None) => { + C2ResponseState::MissStoreError + } }, ); } - let bytes = assemble_if_shared(was_authorized, settings, ¶ms, bytes)?; + let (bytes, assembly_state) = assemble_if_shared( + was_authorized, + !contains_publisher_esi, + settings, + ¶ms, + services, + bytes, + )?; + if let Some(state) = assembly_state { + set_assembly_response_state(&mut response, state); + } let bytes = if was_authorized { encode_complete_body(bytes, response_compression(&response))? } else { @@ -1689,13 +1740,9 @@ pub async fn buffer_publisher_response_async( /// Splices this visitor's slots and bids into the seam, if the mode assembles. /// -/// Uses the same byte split as the hit path, and deliberately **not** the `esi` crate. -/// That crate loses content inside any element larger than its 16 KB `chunk_size`: it -/// empties its buffer before parsing and never restores those bytes when the parser -/// returns `Incomplete`. Next.js streams its RSC payload as a few enormous -/// `self.__next_f.push(...)` scripts, so a real 1.4 MB page came back at 697 KB and the -/// browser showed an error boundary. Raising `chunk_size` moves the threshold rather -/// than removing it. +/// Gives the platform assembler the already-buffered cold document. Fastly resolves a +/// synthetic ESI include with the repaired parser; adapters without an assembler and +/// documents the parser rejects use the same validated byte split as the warm path. /// /// Called *after* [`store_template_if_authorized`], never before: what is stored must be /// the template every visitor shares. @@ -1705,12 +1752,14 @@ pub async fn buffer_publisher_response_async( /// Returns an error if the seam marker is missing or repeated. fn assemble_if_shared( was_authorized: bool, + platform_assembly_allowed: bool, settings: &Settings, params: &OwnedProcessResponseParams, + services: &RuntimeServices, bytes: Vec, -) -> Result, Report> { +) -> Result<(Vec, Option), Report> { if !response_carries_a_seam_marker(was_authorized, settings) { - return Ok(bytes); + return Ok((bytes, None)); } let (head, tail) = split_template_at_seam(&bytes).change_context_lazy(|| { @@ -1720,11 +1769,35 @@ fn assemble_if_shared( })?; let seam = seam_script_for(params); + let platform_result = platform_assembly_allowed.then(|| { + services + .template_assembler() + .assemble(&bytes, seam.as_bytes()) + }); + match platform_result { + Some(Ok(assembled)) + if !assembled + .windows(AD_ASSEMBLY_SEAM.len()) + .any(|window| window == AD_ASSEMBLY_SEAM.as_bytes()) => + { + return Ok((assembled, Some(AssemblyResponseState::EsiParser))); + } + Some(Ok(_)) => { + log::warn!( + "platform template assembler left the shared seam unresolved; using byte-seam fallback" + ); + } + Some(Err(err)) => { + log::warn!("platform template assembly failed: {err}; using byte-seam fallback"); + } + None => {} + } + let mut out = Vec::with_capacity(head.len() + seam.len() + tail.len()); out.extend_from_slice(head); out.extend_from_slice(seam.as_bytes()); out.extend_from_slice(tail); - Ok(out) + Ok((out, Some(AssemblyResponseState::ByteSeamFallback))) } /// Fingerprint of every configuration input plus the compiled browser bundle. @@ -1766,6 +1839,17 @@ fn response_carries_a_seam_marker(was_authorized: bool, settings: &Settings) -> was_authorized && mode_emits_seam_marker(configured_assembly_mode(settings)) } +/// Whether bytes contain an ESI directive that came from publisher content. +/// +/// TS's stored seam is an inert HTML comment, so any ` bool { + bytes + .windows(b"`, or nothing at all. /// /// `seam_ad_slots` is `None` exactly when the ad stack did not run — bot, prefetch, @@ -1958,6 +2042,7 @@ fn build_cached_template_response( enforce_private_no_store(&mut response); set_response_compression(&mut response, reader_compression); set_c2_response_state(&mut response, C2ResponseState::Hit); + set_assembly_response_state(&mut response, AssemblyResponseState::ByteSeam); Ok(response) } @@ -7345,13 +7430,15 @@ mod tests { }; use crate::test_support::tests::crate_test_settings_str; use std::collections::HashMap; - use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; /// A working cache, unlike the recorder above — this one has to actually return /// what it stored, or a hit proves nothing. #[derive(Default)] struct MemoryTemplateCache { entries: Arc>>, + /// Set only after a reservation has committed its reader-neutral bytes. + insert_completed: Arc, /// Every key `get` was called with, so a test can assert what was *asked /// for* rather than only what came back. A lookup that names a schema /// version this binary cannot assemble is the bug; whether the entry @@ -7369,11 +7456,64 @@ mod tests { struct MemoryTemplateReservation { entries: Arc>>, + insert_completed: Arc, stored_keys: Arc>>, stored_max_ages: Arc>>, key: crate::platform::TemplateCacheKey, } + #[derive(Default)] + struct RecordingTemplateAssembler { + calls: AtomicUsize, + fail: AtomicBool, + expected_store_completion: Mutex>>, + } + + impl RecordingTemplateAssembler { + fn require_store_before_call(&self, completed: Arc) { + *self + .expected_store_completion + .lock() + .expect("should lock expected store completion") = Some(completed); + } + } + + impl crate::platform::PlatformTemplateAssembler for RecordingTemplateAssembler { + fn assemble( + &self, + template: &[u8], + fragment: &[u8], + ) -> Result, crate::platform::TemplateAssemblyError> { + self.calls.fetch_add(1, Ordering::Relaxed); + if self + .expected_store_completion + .lock() + .expect("should lock expected store completion") + .as_ref() + .is_some_and(|completed| !completed.load(Ordering::Relaxed)) + { + return Err(crate::platform::TemplateAssemblyError::Failed { + message: "assembler ran before cache insertion completed".to_string(), + }); + } + if self.fail.load(Ordering::Relaxed) { + return Err(crate::platform::TemplateAssemblyError::Failed { + message: "injected assembly failure".to_string(), + }); + } + let (head, tail) = split_template_at_seam(template).map_err(|error| { + crate::platform::TemplateAssemblyError::Failed { + message: error.to_string(), + } + })?; + let mut assembled = Vec::with_capacity(head.len() + fragment.len() + tail.len()); + assembled.extend_from_slice(head); + assembled.extend_from_slice(fragment); + assembled.extend_from_slice(tail); + Ok(assembled) + } + } + impl crate::platform::PlatformTemplateCacheReservation for MemoryTemplateReservation { fn insert( self: Box, @@ -7396,6 +7536,7 @@ mod tests { body, }, ); + self.insert_completed.store(true, Ordering::Relaxed); Ok(()) } @@ -7433,6 +7574,7 @@ mod tests { crate::platform::TemplateCacheReservation::new(Box::new( MemoryTemplateReservation { entries: Arc::clone(&self.entries), + insert_completed: Arc::clone(&self.insert_completed), stored_keys: Arc::clone(&self.stored_keys), stored_max_ages: Arc::clone(&self.stored_max_ages), key: key.clone(), @@ -7480,6 +7622,7 @@ mod tests { body, }, ); + self.insert_completed.store(true, Ordering::Relaxed); Ok(()) } @@ -7554,6 +7697,14 @@ mod tests { .build() } + fn services_with_assembler( + http_client: Arc, + cache: Arc, + assembler: Arc, + ) -> RuntimeServices { + services(http_client, cache).with_template_assembler(assembler) + } + /// Shareable HTML: no `Set-Cookie`, no `Vary`, a public `Cache-Control`. Every /// condition the gate checks is satisfied, so a bypass here would be a bug in /// the wiring rather than in the fixture. @@ -7923,6 +8074,203 @@ mod tests { ); } + #[tokio::test] + async fn only_the_cold_miss_uses_the_platform_assembler() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let assembler = Arc::new(RecordingTemplateAssembler::default()); + assembler.require_store_before_call(Arc::clone(&cache.insert_completed)); + let settings = Arc::new(settings_with_mode("esi")); + let services = services_with_assembler( + Arc::clone(&stub), + Arc::clone(&cache), + Arc::clone(&assembler), + ); + queue_shareable_html(&stub); + + let cold = run(&settings, &services, navigation_request()).await; + assert_eq!( + header_of(&cold, header::HeaderName::from_static("x-ts-assembly")), + Some("esi-parser") + ); + let cold_body = body_of(cold).await; + assert_eq!(assembler.calls.load(Ordering::Relaxed), 1); + + let warm = run(&settings, &services, navigation_request()).await; + assert_eq!( + header_of(&warm, header::HeaderName::from_static("x-ts-assembly")), + Some("byte-seam") + ); + let warm_body = body_of(warm).await; + + assert_eq!( + assembler.calls.load(Ordering::Relaxed), + 1, + "a warm hit must preserve the streaming byte-seam path" + ); + assert_eq!(cold_body, warm_body); + } + + #[tokio::test] + async fn a_platform_assembly_failure_falls_back_to_a_complete_byte_seam() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let assembler = Arc::new(RecordingTemplateAssembler::default()); + assembler.fail.store(true, Ordering::Relaxed); + let settings = Arc::new(settings_with_mode("esi")); + let services = services_with_assembler( + Arc::clone(&stub), + Arc::clone(&cache), + Arc::clone(&assembler), + ); + queue_shareable_html(&stub); + + let response = run(&settings, &services, navigation_request()).await; + assert_eq!( + header_of(&response, header::HeaderName::from_static("x-ts-assembly")), + Some("byte-seam-fallback") + ); + let document = String::from_utf8(body_of(response).await) + .expect("served document should be UTF-8"); + + assert!(document.contains("origin")); + assert!(document.contains("scheduleInitialAdInit")); + assert!(!document.contains(AD_ASSEMBLY_SEAM)); + assert_eq!(assembler.calls.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn publisher_esi_is_never_stored_or_executed() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let assembler = Arc::new(RecordingTemplateAssembler::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services_with_assembler( + Arc::clone(&stub), + Arc::clone(&cache), + Arc::clone(&assembler), + ); + stub.push_response_with_headers( + 200, + b"publisher" + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); + + let response = run(&settings, &services, navigation_request()).await; + assert_eq!( + header_of(&response, header::HeaderName::from_static("x-ts-c2-cache")), + Some("bypass-response") + ); + assert_eq!( + header_of(&response, header::HeaderName::from_static("x-ts-assembly")), + Some("byte-seam-fallback") + ); + let document = String::from_utf8(body_of(response).await) + .expect("served document should be UTF-8"); + + assert!(document.to_ascii_lowercase().contains("")); + assert!(document.contains("scheduleInitialAdInit")); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "publisher-authored ESI must never enter the shared template cache" + ); + assert_eq!( + assembler.calls.load(Ordering::Relaxed), + 0, + "publisher-authored ESI must never reach the platform parser" + ); + } + + #[tokio::test] + async fn ineligible_requests_and_responses_never_use_the_platform_assembler() { + let inline_stub = Arc::new(StubHttpClient::new()); + let inline_cache = Arc::new(MemoryTemplateCache::default()); + let inline_assembler = Arc::new(RecordingTemplateAssembler::default()); + let inline_settings = Arc::new(settings_with_mode("inline")); + let inline_services = services_with_assembler( + Arc::clone(&inline_stub), + Arc::clone(&inline_cache), + Arc::clone(&inline_assembler), + ); + queue_shareable_html(&inline_stub); + + let _ = + body_of(run(&inline_settings, &inline_services, navigation_request()).await).await; + assert_eq!(inline_assembler.calls.load(Ordering::Relaxed), 0); + + let private_stub = Arc::new(StubHttpClient::new()); + let private_cache = Arc::new(MemoryTemplateCache::default()); + let private_assembler = Arc::new(RecordingTemplateAssembler::default()); + let esi_settings = Arc::new(settings_with_mode("esi")); + let private_services = services_with_assembler( + Arc::clone(&private_stub), + Arc::clone(&private_cache), + Arc::clone(&private_assembler), + ); + private_stub.push_response_with_headers( + 200, + b"private origin".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ("set-cookie", "reader=personalized; Path=/"), + ], + ); + + let _ = + body_of(run(&esi_settings, &private_services, navigation_request()).await).await; + assert_eq!(private_assembler.calls.load(Ordering::Relaxed), 0); + + let authenticated_stub = Arc::new(StubHttpClient::new()); + let authenticated_cache = Arc::new(MemoryTemplateCache::default()); + let authenticated_assembler = Arc::new(RecordingTemplateAssembler::default()); + let authenticated_services = services_with_assembler( + Arc::clone(&authenticated_stub), + Arc::clone(&authenticated_cache), + Arc::clone(&authenticated_assembler), + ); + queue_shareable_html(&authenticated_stub); + let mut authenticated_request = navigation_request(); + authenticated_request.headers_mut().insert( + header::AUTHORIZATION, + HeaderValue::from_static("Bearer example-token"), + ); + + let _ = body_of( + run( + &esi_settings, + &authenticated_services, + authenticated_request, + ) + .await, + ) + .await; + assert_eq!(authenticated_assembler.calls.load(Ordering::Relaxed), 0); + + let failed_cache_stub = Arc::new(StubHttpClient::new()); + let failed_cache = Arc::new(MemoryTemplateCache::default()); + failed_cache.fail_lookup.store(true, Ordering::Relaxed); + let failed_cache_assembler = Arc::new(RecordingTemplateAssembler::default()); + let failed_cache_services = services_with_assembler( + Arc::clone(&failed_cache_stub), + Arc::clone(&failed_cache), + Arc::clone(&failed_cache_assembler), + ); + queue_shareable_html(&failed_cache_stub); + + let _ = body_of(run(&esi_settings, &failed_cache_services, navigation_request()).await) + .await; + assert_eq!(failed_cache_assembler.calls.load(Ordering::Relaxed), 0); + } + #[tokio::test] async fn fastly_surrogate_control_still_allows_a_cold_fill_and_warm_hit() { let stub = Arc::new(StubHttpClient::new()); @@ -8227,6 +8575,13 @@ mod tests { ); } + #[test] + fn parser_validation_does_not_change_the_cached_schema() { + assert_eq!(crate::platform::TEMPLATE_SCHEMA_VERSION, 4); + assert_eq!(AD_ASSEMBLY_SEAM, ""); + assert!(!contains_esi_directive(AD_ASSEMBLY_SEAM.as_bytes())); + } + /// Shareable HTML that already contains the seam marker. /// /// The marker is reserved, but publisher content can still contain it. Fresh diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 66ce7b221..531608f8f 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1374,11 +1374,17 @@ formats = [{ width = 728, height = 90 }] - `inline` (default) transforms every origin response and injects the current reader's slots and bids directly. - `esi` opts into a reader-neutral transformed-template cache on Fastly. The - public name is retained for operator continuity; the implementation does not - execute general ESI. It stores identity bytes containing one inert, versioned - comment and performs an exact byte split at that seam. Slots and structured - bids are inserted per request, with the article prefix streamed before the - auction finishes. + cache stores identity bytes containing one inert, versioned comment. On an + authorized cold miss, Fastly replaces that comment in a private working copy + with one synthetic ESI include and resolves it from the already-built reader + state using the pinned `stackpop/esi` parser. No HTTP fragment request occurs. + Warm hits use an exact byte split instead, preserving the fast article-prefix + stream while the auction finishes. + +This is deliberately not general publisher-controlled ESI. A transformed origin +document containing any ` **Execution note:** Implemented inline in the current checkout, without a worktree or +> subagents, as requested. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Use the repaired ESI parser on authorized cold C2 misses without changing the existing warm-hit streaming behavior. + +**Architecture:** C2 retains the inert schema-v4 seam. Core delegates cold assembly through a platform trait; Fastly converts the seam to one synthetic ESI include and resolves it from the already-collected per-reader script. Parser failure falls back to core's validated byte split, while warm hits continue to stream by byte seam. + +**Tech Stack:** Rust 1.95, `wasm32-wasip1`, Fastly Compute/Viceroy, `stackpop/esi` pinned by Git revision, `error-stack`. + +--- + +### Task 1: Restore a platform assembly boundary + +**Files:** + +- Create: `crates/trusted-server-core/src/platform/template_assembly.rs` +- Modify: `crates/trusted-server-core/src/platform/mod.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Test: `crates/trusted-server-core/src/platform/template_assembly.rs` + +- [x] Add a failing object-safety/default-behavior test for `PlatformTemplateAssembler`. +- [x] Run the focused core test and confirm it fails because the boundary is absent. +- [x] Add the trait, error type, unavailable default, runtime service field, builder method, + accessor, and test support. +- [x] Run the focused tests and confirm they pass. + +### Task 2: Delegate only cold-miss assembly + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` + +- [x] Add a recording assembler to the C2 end-to-end tests. +- [x] Add a test asserting one platform call on a cold miss and no additional call on the + subsequent warm hit. +- [x] Add a test asserting platform failure returns a complete byte-seam response. +- [x] Add tests for `x-ts-assembly` values on parser, fallback, and warm paths. +- [x] Run each test first and confirm the expected failure. +- [x] Change `assemble_if_shared` to call the platform assembler after storage, fall back + to the validated byte split on error, and return the assembly method. +- [x] Set `x-ts-assembly` without changing `x-ts-c2-cache` or privacy headers. +- [x] Re-run the C2 end-to-end test module. + +### Task 3: Add the repaired Fastly ESI adapter + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/Cargo.toml` +- Modify: `Cargo.lock` +- Create: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Test: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` + +- [x] Add failing adapter tests for a large Next.js script followed by the seam, an + unexpected publisher ESI directive, an unexpected dispatcher URL, and verbatim + fragment content. +- [x] Run the focused Fastly test filter and confirm the missing module/implementation + fails. +- [x] Pin `https://github.com/stackpop/esi.git` at + `4c53feab4d22ad9a84641b4c46f3f63bc6d197e2`. +- [x] Implement the explicit no-cache/no-DCA ESI configuration and synthetic completed + fragment dispatcher. +- [x] Register `FastlyTemplateAssembler` in per-request runtime services. +- [x] Run the focused Fastly tests and confirm they pass. + +### Task 4: Preserve cache schema and documentation truth + +**Files:** + +- Modify: `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md` +- Modify: `docs/guide/configuration.md` +- Modify: `scripts/c2-local-test.sh` +- Test: `crates/trusted-server-core/src/platform/template_cache.rs` + +- [x] Add/adjust tests proving schema version 4 and the inert stored marker remain + unchanged. +- [x] Extend the local harness to require `esi-parser` on the miss and `byte-seam` on the + hit. +- [x] Update architecture and operator documentation to describe the hybrid path and + pinned fork accurately. +- [x] Run formatting and the harness's static checks. + +### Task 5: Full verification and signed commit + +**Files:** + +- Review every modified file. + +- [x] Run `cargo fmt --all -- --check`. +- [x] Run every target-matched Clippy alias from `CLAUDE.md`. +- [x] Run `cargo test-fastly`, `cargo test-axum`, `cargo test-cloudflare`, and + `cargo test-spin`. +- [x] Run the integration parity test. +- [x] Run JS tests/build/format and docs format. +- [x] Run the C2 local harness when Viceroy and its certificate environment are + available; otherwise report that environmental gap explicitly. +- [x] Run `git diff --check`, inspect staged scope, and confirm no operator configuration + or secrets are staged. +- [x] Create one SSH-signed commit only after every required gate is green. +- [x] Verify the commit signature locally and report the exact commit ID and test counts. diff --git a/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md index 1e030f159..5b593a15d 100644 --- a/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md +++ b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md @@ -6,14 +6,14 @@ stands. **Issue:** IABTechLab/trusted-server#1009 -> **Implementation update, 2026-08-12.** Design C is now the only shared-template -> render path. The operator-facing value remains `assembly_mode = "esi"`, but the -> `esi` crate, native include/subrequest design, executable fragment, and -> `client_fill` comparison arm were removed. Fastly supplies C2; other adapters -> fall back to inline processing. Sections 2–2c and Design B below are retained as -> the investigation history, not as descriptions of current code. The merged and -> hardened contract is -> [the implementation design](./2026-08-12-1009-esi-merge-hardening-design.md). +> **Implementation update, 2026-08-14.** Warm C2 hits still use Design C exactly as +> specified here. Authorized cold misses now also validate the repaired +> `stackpop/esi` parser, pinned by commit: the inert C2 marker becomes one synthetic +> include only in a private working copy, resolved from the already-built reader +> fragment without an HTTP request. Parser failure falls back to the byte seam. See +> [the hybrid implementation design](./2026-08-14-1009-esi-parser-assembly-design.md). +> Sections 2–2c and Design B remain investigation history; the native self-subrequest +> design is still not implemented. --- @@ -214,18 +214,22 @@ it and must stop. The issue's gating decision is: _is Fastly-first acceptable for the flagship perf path, with a portable fallback?_ -Design C keeps assembly portable, but only Fastly currently supplies the shared C2 cache; -the other adapters degrade to their existing inline origin transform. It removes the -`esi` dependency from the critical render path, needs no self-referencing backend, and -avoids a second rendering architecture. - -**ESI is therefore sufficient but unnecessary.** For a single insertion point at a known -location, its parsing generality buys nothing a byte split does not. That is a stronger -answer than validating ESI, and it is the opposite of what the issue expected. - -The existing `esi` mode now selects Design C. Design B is deliberately removed: retaining -a parser and fragment surface for unused generality was not cheap once the real 1.4 MB -Next.js page demonstrated parser truncation. +Design C keeps the latency-critical warm path portable, but only Fastly currently supplies +the shared C2 cache; the other adapters degrade to their existing inline origin transform. +No self-referencing backend or browser-visible fragment surface exists. + +The `esi` mode uses a hybrid implementation. A cold miss is already buffered for cache +insertion, so Fastly runs the repaired ESI parser there to validate the issue's requested +mechanism without adding a new TTFB hold. A warm hit uses Design C's exact byte split, +because parsing the full cached document would buffer the response and expose the auction +at first byte. `X-TS-Assembly: esi-parser` and `X-TS-Assembly: byte-seam` make those two +paths observable. + +The original parser truncation was real: its streaming loop discarded an incomplete +element whenever that element crossed the 16 KiB read boundary. The pinned fork preserves +the incomplete buffer, and an adapter test protects a 120 KiB Next.js-style script. The +stored template remains an inert comment; executable ESI exists only in a request-private +working copy and can dispatch only TS's synthetic completed fragment. ## 7. Sequencing diff --git a/docs/superpowers/specs/2026-08-14-1009-esi-parser-assembly-design.md b/docs/superpowers/specs/2026-08-14-1009-esi-parser-assembly-design.md new file mode 100644 index 000000000..05529b765 --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-1009-esi-parser-assembly-design.md @@ -0,0 +1,153 @@ +# #1009 ESI parser assembly design + +**Date:** 2026-08-14 + +**Status:** Approved implementation delta + +**Base:** `1009-esi-cacheable-root-spec` + +## Goal + +Use the repaired `stackpop/esi` streaming parser for an authorized C2 cold miss while +preserving the current fast, portable byte-seam stream for a warm C2 hit. + +The operator-facing mode remains: + +```toml +[creative_opportunities] +assembly_mode = "esi" +``` + +## Approaches considered + +### 1. Keep byte splitting on every path + +This is the smallest and fastest implementation, and it is the current behavior. It does +not validate that the ESI implementation requested by #1009 works on real publisher HTML. + +### 2. Run the ESI parser on cold and warm responses + +This most literally uses ESI, but the current adapter API returns a complete `String`. +Using it on a warm hit would buffer the whole response and reintroduce the measured +auction-sized TTFB regression. + +### 3. Run ESI on cold misses and byte-stream warm hits + +This is the selected design. A cold miss already buffers the transformed response before +storing it, so ESI adds no new buffering boundary there. A warm hit retains the existing +head-first stream and waits for the auction only at the final seam. + +## Cached representation + +C2 continues to store exactly one inert marker: + +```html + +``` + +The template schema stays at version 4 because the cached bytes do not change. The stored +object never contains executable ESI markup, per-reader slots, bids, cookies, or other +reader state. + +Keeping the inert marker preserves the current collision normalization and makes an +unassembled cached object harmless. It also avoids making an obsolete HTTP fragment route +part of the cache schema. + +## Cold-miss data flow + +1. Fetch and transform the origin response. +2. Normalize and validate the single inert marker. +3. Refuse C2 storage and platform parsing if publisher bytes contain any ``. +8. `stackpop/esi` parses the complete publisher document. Its dispatcher returns the + already-built seam as `PendingFragmentContent::CompletedRequest`; no HTTP request or + nested executor is involved. +9. Compress and serve the assembled response as `private, no-store`. + +The template is stored before assembly. Reversing those operations would cache one +reader's ad state and is forbidden. + +## Warm-hit data flow + +A warm hit does not invoke the ESI parser. It keeps the existing streaming finalizer: + +1. Validate the cached marker before committing headers. +2. Stream the template prefix immediately. +3. Collect the already-dispatched auction at the seam. +4. Stream the reader's slots-and-bids script. +5. Stream the template suffix and finish the encoder. + +This retains the observed millisecond TTFB while the response may continue downloading +until the auction resolves. + +## Platform boundary + +Core regains a small byte-oriented `PlatformTemplateAssembler` trait. Fastly supplies the +ESI-backed implementation. Other adapters receive an unavailable implementation by +default. Core retains the validated original bytes until assembly succeeds, so fallback +never depends on recovering parser output. + +An unavailable or failed platform assembler falls back to the already-validated byte +split for that cold response. This preserves availability and adapter portability without +silently changing the cached object. + +## ESI safety contract + +The Fastly adapter: + +- pins `stackpop/esi` to verified commit + `4c53feab4d22ad9a84641b4c46f3f63bc6d197e2`; +- disables include caching, rendered caching, `edge_control`, DCA inheritance, and nested + includes; +- rejects publisher-authored ESI directives before inserting its synthetic include; +- validates that the dispatcher receives only the synthetic internal include; +- treats fragment bytes as data and never reparses them as ESI; +- requires the inert seam marker exactly once, even though core already validates it; +- verifies parser output is byte-for-byte the original template with only that seam + replaced; any truncation or unrelated parser mutation triggers fallback. + +Any violation returns an assembler error and core uses the safe byte-seam fallback. +Core performs the publisher-ESI rejection before C2 storage as the primary guard; the +adapter repeats it as defense in depth. + +## Request-visible diagnostics + +Assembled C2 responses carry an `x-ts-assembly` header: + +| Path | Value | +| --------------------------------------------------------------------------------------- | -------------------- | +| Authorized cold miss using the repaired parser | `esi-parser` | +| Cold response whose platform parser is unavailable, disallowed, or rejects the document | `byte-seam-fallback` | +| Warm cache hit | `byte-seam` | + +Together with `x-ts-c2-cache`, this lets an operator prove the internal path with two +ordinary requests instead of inferring it from timing. + +## Testing + +Tests must prove: + +- a cold miss invokes the platform assembler once; +- a warm hit does not invoke it again; +- platform failure produces a complete byte-seam response; +- the real ESI adapter resolves the synthetic include after scripts larger than 16 KiB; +- publisher ESI directives are rejected rather than executed; +- nested ESI in the fragment is emitted verbatim; +- the stored template remains reader-neutral and contains the inert schema-v4 marker; +- cold and warm responses both contain populated slots and the same bid state; +- request-visible assembly headers identify each path; +- all existing cache, privacy, policy-header, TTL, compression, and SPA-race tests remain + green. + +## Non-goals + +- No self-referencing Fastly backend. +- No browser-visible fragment request. +- No restoration of `format=fragment` on `/_ts/page-bids`. +- No ESI parsing on warm hits. +- No change to cache TTL policy or the operator configuration shape. diff --git a/scripts/c2-local-test.sh b/scripts/c2-local-test.sh index 275014847..cd987551a 100755 --- a/scripts/c2-local-test.sh +++ b/scripts/c2-local-test.sh @@ -141,6 +141,7 @@ class H(BaseHTTPRequestHandler): self.wfile.write(body) def do_GET(self): + print(f"origin: received GET {self.path}", flush=True) # Compresses when asked, because a real origin does and because a plaintext-only # stub hid a bug that broke the feature end to end: a gzip template has no # findable seam marker, and splicing plaintext bids into a gzip stream gives the @@ -159,6 +160,7 @@ class H(BaseHTTPRequestHandler): self._send(PAGE, "text/html; charset=utf-8", base) def do_POST(self): + print(f"origin: received POST {self.path}", flush=True) n = int(self.headers.get("Content-Length") or 0) if n: self.rfile.read(n) @@ -261,7 +263,7 @@ req() { # req [extra curl args...] "$@" "http://127.0.0.1:$TS_PORT/article" } -origin_gets() { grep -c "GET /article" "$WORK/origin.log" || true; } +origin_gets() { grep -cF "origin: received GET /article" "$WORK/origin.log" || true; } info "Running assertions (mode: $MODE)" BEFORE=$(origin_gets) @@ -307,14 +309,19 @@ check "second request returns 200" "$CODE2" "200" # it. Asserted in both modes: a shared-mode failure that inline shares would otherwise # read as "the fixture never bids" rather than "the seam drops bids". WINNING_BID='\"hb_pb\":\"4.25\"' -# Must stay in step with `AD_ASSEMBLY_SEAM` in publisher.rs. An inert HTML comment, -# not executable ESI markup — nothing parses ESI on the render path any more. +# Must stay in step with `AD_ASSEMBLY_SEAM` in publisher.rs. C2 stores this inert +# comment. The cold response turns it into a synthetic ESI include only in a private +# working copy; the warm response splits these bytes directly. SEAM_MARKER='' c2_state() { awk 'tolower($1) == "x-ts-c2-cache:" { gsub(/\r/, "", $2); print $2 }' "$1" | tail -1 } +assembly_state() { + awk 'tolower($1) == "x-ts-assembly:" { gsub(/\r/, "", $2); print $2 }' "$1" | tail -1 +} + # Shared by the ESI assertions below. check_hit_is_private() { local hdrs @@ -328,12 +335,12 @@ check_hit_is_private() { check_post_reaches_origin() { local before - before=$(grep -c "POST /article" "$WORK/origin.log" || true) + before=$(grep -cF "origin: received POST /article" "$WORK/origin.log" || true) curl -s -o /dev/null -X POST -d 'x=1' -H "Host: ts.example.com" \ -H "Accept-Encoding: gzip" \ "http://127.0.0.1:$TS_PORT/article" check "a POST still reaches the origin" \ - "$(( $(grep -c "POST /article" "$WORK/origin.log" || true) - before ))" "1" + "$(( $(grep -cF "origin: received POST /article" "$WORK/origin.log" || true) - before ))" "1" } if [ "$MODE" = "inline" ]; then @@ -346,6 +353,10 @@ else check "second request is served from cache" "$FETCHES" "1" check "cold request reports a stored miss" "$(c2_state "$WORK/r1.html.headers")" "miss-stored" check "warm request reports a cache hit" "$(c2_state "$WORK/r2.html.headers")" "hit" + check "cold response uses the repaired ESI parser" \ + "$(assembly_state "$WORK/r1.html.headers")" "esi-parser" + check "warm response keeps the streaming byte seam" \ + "$(assembly_state "$WORK/r2.html.headers")" "byte-seam" check "no unresolved seam marker reaches the browser" \ "$(grep -cF "$SEAM_MARKER" "$SERVED" || true)" "0" check "a bids script is present" \ @@ -494,10 +505,10 @@ if [ "$MODE" = "esi" ]; then "$(grep -c 'window\.tsjs' "$SERVED" || true)" cat <<'EOF' - The marker is never visible in page source, in any mode. It exists only inside - the cache; assembly replaces it before the response is sent, on both the miss - and hit paths. `seam marker present: true` above is the evidence that the - stored copy is genuinely reader-agnostic rather than carrying someone's bids. + The marker is never visible in page source. It remains inert in C2. A cold miss + converts it to one synthetic ESI include in a private working copy; a warm hit + byte-splits it directly. Both replace it before sending the response. + `seam marker present: true` above proves the stored copy is reader-agnostic. EOF fi diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 81771383e..83ecbe369 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -192,8 +192,10 @@ auction_timeout_ms = 500 # override via TRUSTED_SERVER__CREATIVE_OPPORTUNITIES_ price_granularity = "dense" # Initial-page delivery mode. `inline` is the default and current production -# behaviour. `esi` is an opt-in Fastly Core Cache experiment: despite the name, -# it uses an inert comment plus exact byte-seam assembly, not an ESI parser. +# behaviour. `esi` is an opt-in Fastly Core Cache experiment. C2 stores an inert +# comment; a cold miss validates the repaired ESI parser in a private working +# copy, while warm hits use the streaming byte seam. No fragment HTTP request or +# publisher-controlled ESI is allowed. # This and the three cache-safety keys below belong in this # [creative_opportunities] table. See docs/guide/configuration.md before enabling. # assembly_mode = "inline" From fa9326900bb43621fdb4202bda5b10de3beb590a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 14 Aug 2026 16:22:38 +0530 Subject: [PATCH 236/395] Merge ESI #1013 into rc/202608 --- .github/workflows/test.yml | 16 + Cargo.lock | 113 +- Cargo.toml | 1 + .../trusted-server-adapter-fastly/Cargo.toml | 2 + .../trusted-server-adapter-fastly/src/app.rs | 40 +- .../src/esi_assembly.rs | 287 + .../trusted-server-adapter-fastly/src/main.rs | 62 +- .../src/template_cache.rs | 505 ++ crates/trusted-server-core/Cargo.toml | 1 + .../benches/html_processor_bench.rs | 8 +- .../src/creative_opportunities.rs | 315 +- .../trusted-server-core/src/html_processor.rs | 147 +- .../src/integrations/datadome/protection.rs | 65 +- .../src/integrations/gpt_bootstrap.js | 6 +- .../src/integrations/gpt_diagnostics.rs | 77 + .../trusted-server-core/src/platform/mod.rs | 13 + .../src/platform/template_assembly.rs | 77 + .../src/platform/template_cache.rs | 1096 +++ .../trusted-server-core/src/platform/types.rs | 74 + crates/trusted-server-core/src/publisher.rs | 7449 ++++++++++++++++- .../src/response_privacy.rs | 112 +- .../trusted-server-js/lib/src/core/types.ts | 12 +- .../lib/src/integrations/gpt/index.ts | 16 +- .../integrations/gpt/gpt_bootstrap.test.ts | 30 + .../gpt/schedule_initial_ad_init.test.ts | 63 + docs/guide/configuration.md | 116 + ...2026-08-08-1009-measurement-and-stage-0.md | 1044 +++ .../2026-08-08-1009-measurement-findings.md | 503 ++ .../2026-08-10-1009-esi-validation-spike.md | 967 +++ .../2026-08-12-1009-esi-merge-hardening.md | 294 + .../2026-08-14-1009-esi-parser-assembly.md | 104 + ...08-esi-cacheable-root-validation-design.md | 864 ++ ...11-1009-streaming-assembly-architecture.md | 260 + ...6-08-12-1009-esi-merge-hardening-design.md | 195 + ...6-08-14-1009-esi-parser-assembly-design.md | 153 + scripts/c2-local-test.sh | 607 ++ trusted-server.example.toml | 25 + 37 files changed, 15276 insertions(+), 443 deletions(-) create mode 100644 crates/trusted-server-adapter-fastly/src/esi_assembly.rs create mode 100644 crates/trusted-server-adapter-fastly/src/template_cache.rs create mode 100644 crates/trusted-server-core/src/platform/template_assembly.rs create mode 100644 crates/trusted-server-core/src/platform/template_cache.rs create mode 100644 docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md create mode 100644 docs/superpowers/plans/2026-08-08-1009-measurement-findings.md create mode 100644 docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md create mode 100644 docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md create mode 100644 docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md create mode 100644 docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md create mode 100644 docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md create mode 100644 docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md create mode 100644 docs/superpowers/specs/2026-08-14-1009-esi-parser-assembly-design.md create mode 100755 scripts/c2-local-test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb0233b03..4f0bb6cfb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,6 +27,11 @@ jobs: run: echo "viceroy-version=$(grep '^viceroy ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT shell: bash + - name: Retrieve Node.js version + id: node-version + run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + shell: bash + - name: Set up Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: @@ -45,9 +50,20 @@ jobs: if: steps.cache-viceroy.outputs.cache-hit != 'true' run: cargo install viceroy --version "${{ steps.viceroy-version.outputs.viceroy-version }}" --locked --force + - name: Use Node.js for the served-seam contract + uses: actions/setup-node@v4 + with: + node-version: ${{ steps.node-version.outputs.node-version }} + - name: Run tests run: cargo test-fastly + - name: Run C2 ESI local harness + run: ./scripts/c2-local-test.sh esi + + - name: Run inline control harness + run: ./scripts/c2-local-test.sh inline + test-axum: name: cargo test (axum native) runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index ed1b9c7a3..5c4748b41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,7 +146,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom", + "nom 7.1.3", "num-traits", "rusticata-macros", "thiserror 1.0.69", @@ -254,6 +254,15 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -573,7 +582,18 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", ] [[package]] @@ -583,7 +603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -916,6 +936,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1041,7 +1070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", @@ -1186,7 +1215,7 @@ checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" dependencies = [ "asn1-rs", "displaydoc", - "nom", + "nom 7.1.3", "num-bigint", "num-traits", "rusticata-macros", @@ -1690,6 +1719,26 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "esi" +version = "0.7.1" +source = "git+https://github.com/stackpop/esi.git?rev=4c53feab4d22ad9a84641b4c46f3f63bc6d197e2#4c53feab4d22ad9a84641b4c46f3f63bc6d197e2" +dependencies = [ + "atoi", + "base64", + "bytes", + "chrono", + "fastly", + "html-escape", + "log", + "md5", + "nom 8.0.0", + "percent-encoding", + "rand 0.10.2", + "regex", + "thiserror 2.0.18", +] + [[package]] name = "etcetera" version = "0.10.0" @@ -2034,6 +2083,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -2188,6 +2238,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "html-escape" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9356095b4b41197bba32173600e1582792cda618f65d12f68e2e77d273413c5" + [[package]] name = "html5ever" version = "0.35.0" @@ -2914,6 +2970,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" +[[package]] +name = "md5" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" + [[package]] name = "memchr" version = "2.8.2" @@ -2984,6 +3046,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "num" version = "0.4.3" @@ -3477,7 +3548,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -3775,6 +3846,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20 0.10.1", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3813,6 +3895,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rcgen" version = "0.13.2" @@ -4079,7 +4167,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -4494,7 +4582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -4506,7 +4594,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.9.0", "opaque-debug", ] @@ -4518,7 +4606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -5286,9 +5374,11 @@ dependencies = [ "base64", "bytes", "chrono", + "derive_more", "edgezero-adapter-fastly", "edgezero-core", "error-stack", + "esi", "fastly", "fern", "futures", @@ -5386,6 +5476,7 @@ dependencies = [ "hex", "hmac", "http", + "httpdate", "iab_gpp", "jose-jwk", "log", @@ -6338,7 +6429,7 @@ dependencies = [ "data-encoding", "der-parser", "lazy_static", - "nom", + "nom 7.1.3", "oid-registry", "ring", "rusticata-macros", diff --git a/Cargo.toml b/Cargo.toml index a5a63ca3a..ab5638f5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,6 +71,7 @@ getrandom = "0.2" hex = "0.4.3" hmac = "0.12.1" http = "1.4.0" +httpdate = "1.0.3" http-body-util = "0.1" hyper = "1" hyper-util = "0.1" diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index b6bc0f1a1..3d42ae388 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -15,9 +15,11 @@ async-trait = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } +derive_more = { workspace = true } edgezero-adapter-fastly = { workspace = true, features = ["fastly"] } edgezero-core = { workspace = true } error-stack = { workspace = true } +esi = { git = "https://github.com/stackpop/esi.git", rev = "4c53feab4d22ad9a84641b4c46f3f63bc6d197e2" } fastly = { workspace = true } fern = { workspace = true } futures = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index d6090c983..6be93b4b3 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -257,6 +257,11 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime .config_store(Arc::new(FastlyPlatformConfigStore)) .secret_store(Arc::new(FastlyPlatformSecretStore)) .kv_store(Arc::clone(&state.default_kv_store)) + // Spike-only (#1009). Constructed unconditionally, but only read when the + // assembly mode is a shared-template one — which defaults to Inline, so this + // is inert until an operator opts in. + .template_cache(Arc::new(crate::template_cache::FastlyTemplateCache::new())) + .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) .backend(Arc::new(FastlyPlatformBackend)) .http_client(Arc::new(FastlyPlatformHttpClient)) .geo(Arc::new(FastlyPlatformGeo)) @@ -1239,12 +1244,15 @@ mod tests { use super::{ AppState, NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, - TrustedServerApp, build_state_from_settings, startup_error_router, + TrustedServerApp, build_per_request_services, build_state_from_settings, + startup_error_router, }; use bytes::Bytes; use edgezero_core::body::Body; + use edgezero_core::context::RequestContext; use edgezero_core::http::{Method, Response, StatusCode, header, request_builder}; use edgezero_core::key_value_store::NoopKvStore; + use edgezero_core::params::PathParams; use edgezero_core::router::RouterService; use std::net::{IpAddr, Ipv4Addr}; use std::sync::Mutex; @@ -1379,6 +1387,36 @@ mod tests { TrustedServerApp::routes_for_state(&state) } + #[test] + fn per_request_services_register_the_fastly_template_assembler() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + let context = RequestContext::new( + empty_request(Method::GET, "/article"), + PathParams::default(), + ); + + let services = build_per_request_services(&state, &context); + let template = format!( + "article{}", + trusted_server_core::publisher::AD_ASSEMBLY_SEAM + ); + let fragment = b""; + let assembled = services + .template_assembler() + .assemble(template.as_bytes(), fragment) + .expect("Fastly services should provide ESI assembly"); + + assert_eq!( + assembled, + template + .replace( + trusted_server_core::publisher::AD_ASSEMBLY_SEAM, + std::str::from_utf8(fragment).expect("fragment should be UTF-8") + ) + .into_bytes() + ); + } + /// Builds a router whose `AppState` uses a registry containing the given /// request filters (and no routes), so dispatch-level request-filter /// behavior can be exercised without a real integration. diff --git a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs new file mode 100644 index 000000000..4a92c6f78 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs @@ -0,0 +1,287 @@ +//! Fastly cold-response assembly backed by the repaired `stackpop/esi` parser. +//! +//! C2 stores an inert marker. This module creates one synthetic ESI include only in a +//! request-private working copy, resolves it from an already-built fragment, and never +//! performs an HTTP request. + +use std::io::Cursor; + +use esi::{CacheConfig, Configuration, DcaMode, PendingFragmentContent, Processor}; +use fastly::http::StatusCode; +use fastly::{Request, Response}; +use trusted_server_core::platform::{PlatformTemplateAssembler, TemplateAssemblyError}; +use trusted_server_core::publisher::AD_ASSEMBLY_SEAM; + +const INTERNAL_FRAGMENT_PATH: &str = "/_ts/internal/reader-ad-state"; +const SYNTHETIC_ESI_INCLUDE: &[u8] = b""; + +/// Why the Fastly ESI adapter refused or failed to assemble a document. +#[derive(Debug, derive_more::Display)] +enum EsiAssemblyError { + /// The inert seam marker was missing or repeated. + #[display("expected exactly one inert seam marker, found {count}")] + InvalidMarkerCount { count: usize }, + /// Publisher bytes contained ESI instructions outside TS's synthetic seam. + #[display("publisher-authored ESI directives are not allowed")] + PublisherEsiDirective, + /// The parser dispatched a URL other than TS's one synthetic fragment. + #[display("unexpected fragment request path `{path}` (query present: {has_query})")] + UnexpectedFragmentRequest { path: String, has_query: bool }, + /// The pinned parser could not process the document. + #[display("ESI processing failed: {message}")] + Processing { message: String }, + /// The parser changed bytes outside the one synthetic include. + #[display("ESI output was not an exact seam substitution")] + OutputMismatch, +} + +impl core::error::Error for EsiAssemblyError {} + +/// ESI configuration with every cache- and recursion-sensitive option explicit. +fn assembly_configuration() -> Configuration { + Configuration::default() + .with_escaped(false) + .with_default_dca(DcaMode::None) + .with_inherit_parent_dca(false) + .with_max_include_depth(1) + .with_edge_control(false) + .with_caching(CacheConfig { + is_includes_cacheable: false, + includes_default_ttl: None, + includes_force_ttl: None, + is_rendered_cacheable: false, + rendered_cache_control: false, + rendered_ttl: None, + }) +} + +fn contains_esi_directive(bytes: &[u8]) -> bool { + bytes + .windows(b" Result<(Vec, usize), EsiAssemblyError> { + let marker = AD_ASSEMBLY_SEAM.as_bytes(); + let positions = template + .windows(marker.len()) + .enumerate() + .filter_map(|(at, window)| (window == marker).then_some(at)) + .collect::>(); + if positions.len() != 1 { + return Err(EsiAssemblyError::InvalidMarkerCount { + count: positions.len(), + }); + } + if contains_esi_directive(template) { + return Err(EsiAssemblyError::PublisherEsiDirective); + } + + let at = positions[0]; + let mut working = + Vec::with_capacity(template.len() - marker.len() + SYNTHETIC_ESI_INCLUDE.len()); + working.extend_from_slice(&template[..at]); + working.extend_from_slice(SYNTHETIC_ESI_INCLUDE); + working.extend_from_slice(&template[at + marker.len()..]); + Ok((working, at)) +} + +fn completed_fragment_response( + request: &Request, + fragment: &[u8], +) -> Result { + let path = request.get_path().to_string(); + let has_query = request.get_url().query().is_some(); + if path != INTERNAL_FRAGMENT_PATH || has_query { + return Err(EsiAssemblyError::UnexpectedFragmentRequest { path, has_query }); + } + + Ok(PendingFragmentContent::CompletedRequest(Box::new( + Response::from_status(StatusCode::OK) + .with_header( + fastly::http::header::CONTENT_TYPE, + "text/html; charset=utf-8", + ) + .with_body(fragment.to_vec()), + ))) +} + +fn assemble_with_observer( + template: &[u8], + fragment: &[u8], + on_dispatch: F, +) -> Result, EsiAssemblyError> +where + F: Fn() + 'static, +{ + let (working, seam_at) = template_with_synthetic_include(template)?; + let fragment_len = fragment.len(); + let fragment_response = fragment.to_vec(); + let dispatcher = move |request, _index| { + on_dispatch(); + completed_fragment_response(&request, &fragment_response) + .map_err(|error| esi::ESIError::FragmentRequestError(error.to_string())) + }; + let mut processor = Processor::new(None, assembly_configuration()); + let mut output = Vec::with_capacity(template.len() + fragment_len); + processor + .process_stream(Cursor::new(working), &mut output, Some(&dispatcher), None) + .map_err(|error| EsiAssemblyError::Processing { + message: error.to_string(), + })?; + let expected_len = template.len() - AD_ASSEMBLY_SEAM.len() + fragment_len; + let output_tail_at = seam_at + fragment_len; + let template_tail_at = seam_at + AD_ASSEMBLY_SEAM.len(); + if output.len() != expected_len + || output[..seam_at] != template[..seam_at] + || &output[seam_at..output_tail_at] != fragment + || output[output_tail_at..] != template[template_tail_at..] + { + return Err(EsiAssemblyError::OutputMismatch); + } + Ok(output) +} + +fn assemble(template: &[u8], fragment: &[u8]) -> Result, EsiAssemblyError> { + assemble_with_observer(template, fragment, || {}) +} + +/// Fastly implementation of the core cold-response assembly boundary. +pub struct FastlyTemplateAssembler; + +impl PlatformTemplateAssembler for FastlyTemplateAssembler { + fn assemble(&self, template: &[u8], fragment: &[u8]) -> Result, TemplateAssemblyError> { + assemble(template, fragment).map_err(|error| TemplateAssemblyError::Failed { + message: error.to_string(), + }) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + use trusted_server_core::publisher::AD_ASSEMBLY_SEAM; + + const FRAGMENT: &[u8] = b""; + + fn template(body: &str) -> Vec { + format!("{body}{AD_ASSEMBLY_SEAM}").into_bytes() + } + + #[test] + fn a_script_larger_than_the_parser_chunk_survives_exactly() { + let script = format!( + "", + "x".repeat(120_000) + ); + let document = template(&script); + let dispatches = Arc::new(AtomicUsize::new(0)); + let observed_dispatches = Arc::clone(&dispatches); + + let assembled = assemble_with_observer(&document, FRAGMENT, move || { + observed_dispatches.fetch_add(1, Ordering::Relaxed); + }) + .expect("should assemble a document with a large script"); + + let seam_at = document + .windows(AD_ASSEMBLY_SEAM.len()) + .position(|window| window == AD_ASSEMBLY_SEAM.as_bytes()) + .expect("should find seam"); + let mut expected = Vec::new(); + expected.extend_from_slice(&document[..seam_at]); + expected.extend_from_slice(FRAGMENT); + expected.extend_from_slice(&document[seam_at + AD_ASSEMBLY_SEAM.len()..]); + + assert_eq!( + assembled, expected, + "ESI must alter only the synthetic seam" + ); + assert_eq!(dispatches.load(Ordering::Relaxed), 1); + } + + #[test] + fn missing_and_repeated_markers_are_rejected_before_parsing() { + let missing = assemble(b"plain", FRAGMENT) + .expect_err("should reject a missing marker"); + let repeated = assemble( + format!("{AD_ASSEMBLY_SEAM}{AD_ASSEMBLY_SEAM}").as_bytes(), + FRAGMENT, + ) + .expect_err("should reject repeated markers"); + + assert!(matches!( + missing, + EsiAssemblyError::InvalidMarkerCount { count: 0 } + )); + assert!(matches!( + repeated, + EsiAssemblyError::InvalidMarkerCount { count: 2 } + )); + } + + #[test] + fn every_publisher_esi_directive_form_is_rejected_case_insensitively() { + for directive in [ + "", + "secret", + "x", + "$(HTTP_HOST)", + "text", + "", + ] { + let error = assemble(&template(directive), FRAGMENT) + .expect_err("should reject publisher-authored ESI"); + + assert!(matches!(error, EsiAssemblyError::PublisherEsiDirective)); + } + } + + #[test] + fn fragment_esi_is_emitted_verbatim_and_never_reparsed() { + let fragment = b""; + + let assembled = assemble(&template("article"), fragment).expect("should assemble"); + + assert!( + assembled + .windows(fragment.len()) + .any(|window| window == fragment), + "fragment bytes must remain data" + ); + } + + #[test] + fn dispatcher_rejects_every_url_except_the_synthetic_internal_one() { + let unexpected = fastly::Request::get("https://example.com/not-the-seam"); + let with_query = + fastly::Request::get("https://example.com/_ts/internal/reader-ad-state?publisher=1"); + + assert!(matches!( + completed_fragment_response(&unexpected, FRAGMENT), + Err(EsiAssemblyError::UnexpectedFragmentRequest { .. }) + )); + assert!(matches!( + completed_fragment_response(&with_query, FRAGMENT), + Err(EsiAssemblyError::UnexpectedFragmentRequest { .. }) + )); + } + + #[test] + fn configuration_cannot_cache_or_reparse_reader_state() { + let configuration = assembly_configuration(); + + assert!(!configuration.cache.is_includes_cacheable); + assert!(configuration.cache.includes_default_ttl.is_none()); + assert!(configuration.cache.includes_force_ttl.is_none()); + assert!(!configuration.cache.is_rendered_cacheable); + assert!(!configuration.cache.rendered_cache_control); + assert!(configuration.cache.rendered_ttl.is_none()); + assert_eq!(configuration.default_dca, DcaMode::None); + assert!(!configuration.inherit_parent_dca); + assert_eq!(configuration.max_include_depth, 1); + assert!(!configuration.enable_edge_control); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 39d35b198..07c042ea8 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -29,11 +29,13 @@ mod app; mod backend; mod compat; mod ec_kv; +mod esi_assembly; mod logging; mod management_api; mod middleware; mod platform; mod rate_limiter; +mod template_cache; mod tinybird; use crate::app::{EcFinalizeState, TrustedServerApp, load_settings_from_config_store}; @@ -328,14 +330,7 @@ fn send_edgezero_response( mut response: HttpResponse, request_filter_effects: Option<&RequestFilterEffects>, ) { - if let Some(effects) = request_filter_effects { - effects.apply_to_response(&mut response); - } - - // Final cache guard: EC finalization and request-filter effects may have - // added a per-user Set-Cookie after `apply_finalize_headers` ran, so - // re-apply the privacy downgrade before send. - crate::middleware::enforce_set_cookie_cache_privacy(&mut response); + apply_terminal_response_effects(&mut response, request_filter_effects); let (parts, body) = response.into_parts(); @@ -364,6 +359,26 @@ fn send_edgezero_response( } } +/// Apply every late response mutation, then restore privacy invariants before headers commit. +fn apply_terminal_response_effects( + response: &mut HttpResponse, + request_filter_effects: Option<&RequestFilterEffects>, +) { + let must_remain_private = + trusted_server_core::response_privacy::is_private_or_no_store(response.headers()); + if let Some(effects) = request_filter_effects { + effects.apply_to_response(response); + } + if must_remain_private { + trusted_server_core::response_privacy::enforce_private_no_store(response); + } + + // Final cache guard: EC finalization and request-filter effects may have + // added a per-user Set-Cookie after `apply_finalize_headers` ran, so + // re-apply the privacy downgrade before send. + crate::middleware::enforce_set_cookie_cache_privacy(response); +} + const FALLBACK_UNAVAILABLE: &str = "unavailable"; const FALLBACK_NOT_SENT: &str = "not sent"; const FALLBACK_NONE: &str = "none"; @@ -485,6 +500,7 @@ mod tests { use edgezero_core::http::HeaderValue; use edgezero_core::http::response_builder; use fastly::mime; + use trusted_server_core::integrations::HeaderMutation; fn test_settings() -> Settings { Settings::from_toml( @@ -557,6 +573,36 @@ mod tests { ); } + #[test] + fn late_filter_effects_cannot_make_an_assembled_response_public() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .header("etag", "\"reader-document\"") + .body(EdgeBody::empty()) + .expect("should build response"); + let effects = RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![ + HeaderMutation::set("cache-control", "public, s-maxage=3600"), + HeaderMutation::set("surrogate-control", "max-age=3600"), + HeaderMutation::set("cdn-cache-control", "public, max-age=3600"), + ], + }; + + apply_terminal_response_effects(&mut response, Some(&effects)); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("private, no-store") + ); + assert!(response.headers().get("surrogate-control").is_none()); + assert!(response.headers().get("cdn-cache-control").is_none()); + assert!(response.headers().get("etag").is_none()); + } + #[test] #[allow(clippy::panic)] fn entry_point_finalize_skips_geo_lookup_for_401() { diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs new file mode 100644 index 000000000..3f168f574 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -0,0 +1,505 @@ +//! Fastly Core Cache backing for the shared transformed-template cache (C2). +//! +//! Only the Fastly adapter implements this; every other adapter uses +//! `UnavailableTemplateCache`, so the ESI assembly mode stays portable and only +//! the caching is Fastly-only. +//! +//! **Why Core Cache and not read-through caching.** Read-through with `after_send` + +//! `set_body_transform` looks like a better fit — it keeps HTTP semantics and derives +//! TTL and surrogate keys from origin headers for free. It is unreachable here: +//! Viceroy 0.17 stubs the entire HTTP Cache ABI and the SDK converts that into a +//! *send error*, so setting `after_send` makes every publisher origin fetch fail +//! under `fastly compute serve`, `cargo test-fastly` and the parity suite. It is also +//! silently dead whenever the origin request is in pass mode, and its closure bounds +//! (`Fn + Send + Sync`) are incompatible with a platform layer that is `!Send` by +//! construction. Recorded in the spike plan's Task 3 Step 4 so nobody re-proposes it. +//! +//! Spike-only. Remove with the spike. + +use fastly::cache::core::{CacheKey, Found, Transaction}; +use std::io::Write as _; +use std::time::Duration; +use trusted_server_core::platform::{ + PlatformTemplateCache, PlatformTemplateCacheReservation, TemplateCacheError, TemplateCacheKey, + TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, + TemplateMetadata, +}; + +/// Surrogate key attached to every stored template, so a single purge clears them +/// all. This is the rollback lever: without it, backing out a bad template means +/// waiting for the TTL. +const PURGE_ALL_SURROGATE_KEY: &str = "ts-template"; + +/// Fastly Core Cache implementation of the C2 template cache. +#[derive(Default)] +pub struct FastlyTemplateCache; + +impl FastlyTemplateCache { + /// Create the Fastly Core Cache implementation. + /// + /// Entry lifetime is supplied per insert after core validates origin freshness + /// and applies the operator's configured safety ceiling. + #[must_use] + pub const fn new() -> Self { + Self + } +} + +fn backend_error(message: impl Into) -> TemplateCacheError { + TemplateCacheError::Backend { + message: message.into(), + } +} + +enum ReadFoundError { + Invalid(TemplateCacheMiss), + Backend(TemplateCacheError), +} + +fn read_found(found: &Found, key: &TemplateCacheKey) -> Result { + if found.is_stale() { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::NotFound)); + } + + let metadata = TemplateMetadata::decode(&found.user_metadata()).ok_or( + ReadFoundError::Invalid(TemplateCacheMiss::UnreadableMetadata), + )?; + if metadata.schema_version != key.schema_version { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::SchemaMismatch)); + } + if found + .known_length() + .is_some_and(|length| length != metadata.body_len) + { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::Truncated)); + } + + let body = found + .to_stream() + .map_err(|error| { + ReadFoundError::Backend(backend_error(format!( + "opening cached template body failed: {error:?}" + ))) + })? + .into_bytes(); + if body.len() as u64 != metadata.body_len { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::Truncated)); + } + Ok(TemplateEntry { metadata, body }) +} + +struct FastlyTemplateReservation { + transaction: Transaction, + surrogate_keys: Vec, +} + +impl PlatformTemplateCacheReservation for FastlyTemplateReservation { + fn insert( + self: Box, + metadata: &TemplateMetadata, + body: Vec, + max_age: Duration, + ) -> Result<(), TemplateCacheError> { + if metadata.body_len != body.len() as u64 { + return Err(backend_error(format!( + "metadata body_len {} does not match the {} bytes supplied", + metadata.body_len, + body.len() + ))); + } + + let mut writer = self + .transaction + .insert(max_age) + .surrogate_keys(self.surrogate_keys.iter().map(String::as_str)) + .known_length(body.len() as u64) + .user_metadata(metadata.encode().into()) + .execute() + .map_err(|e| backend_error(format!("cache insert failed: {e:?}")))?; + writer + .write_all(&body) + .map_err(|e| backend_error(format!("writing template body failed: {e}")))?; + writer + .finish() + .map_err(|e| backend_error(format!("finishing the cached template failed: {e}")))?; + Ok(()) + } + + fn cancel(self: Box) -> Result<(), TemplateCacheError> { + self.transaction + .cancel_insert_or_update() + .map_err(|e| backend_error(format!("cancelling cache reservation failed: {e:?}"))) + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformTemplateCache for FastlyTemplateCache { + async fn lookup_or_reserve( + &self, + key: &TemplateCacheKey, + ) -> Result { + let transaction = Transaction::lookup(CacheKey::from(key.to_cache_key().into_bytes())) + .execute() + .map_err(|e| backend_error(format!("transactional lookup failed: {e:?}")))?; + + if transaction.must_insert_or_update() { + return Ok(TemplateCacheLookup::Reserved( + TemplateCacheReservation::new(Box::new(FastlyTemplateReservation { + transaction, + surrogate_keys: key.surrogate_keys(), + })), + )); + } + + let found = transaction.found().ok_or_else(|| { + backend_error("transaction returned neither a hit nor an insert obligation") + })?; + Ok(match read_found(&found, key) { + Ok(entry) => TemplateCacheLookup::Hit(entry), + Err(ReadFoundError::Invalid(miss)) => TemplateCacheLookup::Invalid(miss), + Err(ReadFoundError::Backend(error)) => return Err(error), + }) + } + + async fn get(&self, key: &TemplateCacheKey) -> Result { + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + // A plain lookup, not a transaction: a read that does not intend to insert + // must not take an insert obligation it will never discharge, which would + // block every other client waiting on the same key until they time out. + let found = fastly::cache::core::lookup(cache_key) + .execute() + .map_err(|_| TemplateCacheMiss::NotFound)? + .ok_or(TemplateCacheMiss::NotFound)?; + + read_found(&found, key).map_err(|error| match error { + ReadFoundError::Invalid(miss) => miss, + ReadFoundError::Backend(error) => { + // This legacy method cannot expose a backend error. Production uses + // `lookup_or_reserve`, which preserves it for bounded diagnostics. + log::warn!("c2_template_cache legacy read failed: {error}"); + TemplateCacheMiss::NotFound + } + }) + } + + async fn put( + &self, + key: &TemplateCacheKey, + metadata: &TemplateMetadata, + body: Vec, + max_age: Duration, + ) -> Result<(), TemplateCacheError> { + if metadata.body_len != body.len() as u64 { + return Err(backend_error(format!( + "metadata body_len {} does not match the {} bytes supplied; storing \ + this would make every read a truncation miss", + metadata.body_len, + body.len() + ))); + } + + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + // Transactional insert so a cold key under load transforms once rather than + // once per concurrent request. + let tx = Transaction::lookup(cache_key) + .execute() + .map_err(|e| backend_error(format!("transactional lookup failed: {e:?}")))?; + + // Order matters. A STALE entry sets *both* `found()` and + // `must_insert_or_update()`. Testing `found()` first would return early on + // the stale bytes and never discharge the obligation, leaving every + // concurrent waiter blocked until timeout. + if !tx.must_insert_or_update() { + // Someone else already inserted a fresh entry. Nothing to do, and + // nothing to discharge. + return Ok(()); + } + + // `Transaction::insert` takes `self`, so from here there is no handle left to + // cancel the insert with. A write that fails part-way therefore cannot be + // retracted — which is why `TemplateMetadata::body_len` exists and `get` + // checks it. The metadata is written before the body, so a truncated entry + // still carries the length it was supposed to have. + let surrogate_keys = key.surrogate_keys(); + let mut writer = tx + .insert(max_age) + .surrogate_keys(surrogate_keys.iter().map(String::as_str)) + .user_metadata(metadata.encode().into()) + .execute() + .map_err(|e| backend_error(format!("cache insert failed: {e:?}")))?; + + if let Err(e) = writer.write_all(&body) { + // Deliberately not calling `finish()`. An unfinished entry has no known + // length, and even if it is observable, `get`'s length check rejects it. + return Err(backend_error(format!("writing template body failed: {e}"))); + } + + // Required. Without it the object never completes and its length stays + // unknown, so readers see a partial or absent entry. + writer + .finish() + .map_err(|e| backend_error(format!("finishing the cached template failed: {e}")))?; + + Ok(()) + } + + async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError> { + fastly::http::purge::purge_surrogate_key(&key.url_surrogate_key()) + .map_err(|e| backend_error(format!("purging invalid template failed: {e:?}"))) + } + + async fn purge_all(&self) -> Result<(), TemplateCacheError> { + fastly::http::purge::purge_surrogate_key(PURGE_ALL_SURROGATE_KEY) + .map_err(|e| backend_error(format!("purging templates failed: {e:?}"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use trusted_server_core::creative_opportunities::AssemblyMode; + use trusted_server_core::platform::TEMPLATE_SCHEMA_VERSION; + + /// Distinct per test, so tests sharing the process cache cannot collide. + fn key(url: &str) -> TemplateCacheKey { + TemplateCacheKey { + url: url.to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + origin_identity: "https://origin.example.com\0origin.example.com".to_string(), + assembly_mode: AssemblyMode::Esi, + vary_values: vec![trusted_server_core::platform::VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }], + template_fingerprint: "fp".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + } + } + + fn metadata_for(body: &[u8]) -> TemplateMetadata { + TemplateMetadata { + policy_headers: Vec::new(), + content_encoding: "identity".to_string(), + content_type: "text/html; charset=utf-8".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: body.len() as u64, + } + } + + /// The trait is `async_trait(?Send)` and this crate has no async test runtime, + /// so drive the futures directly. + fn run(fut: impl core::future::Future) -> T { + futures::executor::block_on(fut) + } + + fn cache() -> FastlyTemplateCache { + FastlyTemplateCache::new() + } + + #[test] + fn a_stored_template_reads_back_intact() { + let cache = cache(); + let key = key("https://example.com/roundtrip"); + let body = b"template".to_vec(); + let metadata = metadata_for(&body); + + run(cache.put(&key, &metadata, body.clone(), Duration::from_secs(60))) + .expect("should store"); + + let entry = run(cache.get(&key)).expect("should read back"); + assert_eq!(entry.body, body, "bytes must survive the round trip"); + assert_eq!(entry.metadata, metadata, "metadata must survive too"); + } + + #[test] + fn transactional_lookup_reserves_before_insert_then_hits() { + let cache = cache(); + let key = key("https://example.com/pre-origin-reservation"); + let body = b"collapsed".to_vec(); + let metadata = metadata_for(&body); + + let reservation = match run(cache.lookup_or_reserve(&key)).expect("lookup should work") { + TemplateCacheLookup::Reserved(reservation) => reservation, + _ => panic!("a cold transactional lookup must assign the insert obligation"), + }; + reservation + .insert(&metadata, body.clone(), Duration::from_secs(17)) + .expect("reservation should insert"); + + match run(cache.lookup_or_reserve(&key)).expect("warm lookup should work") { + TemplateCacheLookup::Hit(entry) => assert_eq!(entry.body, body), + _ => panic!("the next transactional lookup must see the inserted template"), + } + } + + #[test] + fn an_absent_key_is_a_miss_not_an_error() { + let miss = + run(cache().get(&key("https://example.com/never-stored"))).expect_err("should miss"); + assert_eq!(miss, TemplateCacheMiss::NotFound); + } + + #[test] + fn a_different_assembly_mode_does_not_read_the_same_entry() { + // The arms emit different bytes. If they shared an entry, one would serve + // the other's template. + let cache = cache(); + let esi = key("https://example.com/mode-split"); + let mut inline = esi.clone(); + inline.assembly_mode = AssemblyMode::Inline; + + let body = b"esi-template".to_vec(); + run(cache.put(&esi, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + + assert_eq!( + run(cache.get(&inline)).err(), + Some(TemplateCacheMiss::NotFound), + "inline must not read the ESI arm's template" + ); + } + + #[test] + fn a_schema_bump_reads_a_miss_rather_than_a_stale_shape() { + let cache = cache(); + let key_v1 = key("https://example.com/schema"); + let body = b"old-shape".to_vec(); + run(cache.put(&key_v1, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + + // A deploy that changes the transform bumps the constant. The old entry must + // not be assembled against. + let mut key_v2 = key_v1.clone(); + key_v2.schema_version = TEMPLATE_SCHEMA_VERSION + 1; + + assert_eq!( + run(cache.get(&key_v2)).err(), + Some(TemplateCacheMiss::NotFound), + "a bumped schema changes the key, so the old entry is simply not found" + ); + } + + #[test] + fn a_stale_but_present_entry_reads_as_a_miss_rather_than_being_served() { + // Stale-while-revalidate is a real option and deliberately not taken: it is a + // state machine `cache::core` does not implement for you, and serving stale here + // means serving a template built by an older transform or an older JS bundle. + // + // The entry has to be *present and stale*, not merely expired. A zero TTL with no + // `stale_while_revalidate` window is simply absent, so a test written that way + // passes without ever reaching `is_stale()` — verified: reverting the staleness + // check left that version green. The revalidate window is what keeps the object + // readable while stale, so this actually exercises the branch. + let key = key("https://example.com/stale"); + let body = b"stale-template".to_vec(); + let metadata = metadata_for(&body); + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + let mut writer = fastly::cache::core::insert(cache_key, Duration::from_secs(0)) + .stale_while_revalidate(Duration::from_secs(60)) + .user_metadata(metadata.encode().into()) + .execute() + .expect("should begin insert"); + writer.write_all(&body).expect("should write body"); + writer.finish().expect("should finish insert"); + + let miss = run(cache().get(&key)).expect_err("a stale template must not be served"); + assert_eq!(miss, TemplateCacheMiss::NotFound); + } + + #[test] + fn purge_all_clears_stored_templates() { + // The rollback lever. Without this, backing out a bad template means waiting + // for the TTL. + let cache = cache(); + let key = key("https://example.com/purge"); + let body = b"template".to_vec(); + run(cache.put(&key, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + run(cache.get(&key)).expect("should be present before purge"); + + run(cache.purge_all()).expect("should purge"); + + assert!( + run(cache.get(&key)).is_err(), + "purge must clear the template, or rollback is TTL-bound" + ); + } + + #[test] + fn a_second_put_on_a_fresh_entry_is_a_no_op() { + // Exercises the `must_insert_or_update` early return: a concurrent writer + // that finds a fresh entry must neither error nor overwrite. + let cache = cache(); + let key = key("https://example.com/second-put"); + let first = b"first".to_vec(); + run(cache.put( + &key, + &metadata_for(&first), + first.clone(), + Duration::from_secs(60), + )) + .expect("first put stores"); + + let second = b"second".to_vec(); + run(cache.put( + &key, + &metadata_for(&second), + second, + Duration::from_secs(60), + )) + .expect("second put should be a no-op, not an error"); + + assert_eq!( + run(cache.get(&key)).expect("should read").body, + first, + "a fresh entry must not be overwritten by a racing writer" + ); + } + + #[test] + fn the_cache_round_trips_through_the_platform_trait_object() { + // Every other test here calls `FastlyTemplateCache` concretely. The publisher + // never does — it reaches the cache as a `dyn PlatformTemplateCache` behind + // `RuntimeServices`. That join is what `app.rs` wires, and until this test it + // was only type-checked, never executed. + let cache: std::sync::Arc = std::sync::Arc::new(cache()); + let key = key("https://example.com/via-trait-object"); + let body = b"template".to_vec(); + + run(cache.put( + &key, + &metadata_for(&body), + body.clone(), + Duration::from_secs(60), + )) + .expect("should store"); + + assert_eq!( + run(cache.get(&key)).expect("should read back").body, + body, + "the trait object must reach the same Core Cache the concrete type does" + ); + } + + #[test] + fn a_length_mismatch_is_refused_at_write_rather_than_stored() { + // Storing metadata whose length disagrees with the body would make every + // subsequent read a truncation miss — a cache that silently never hits. + // Catch it at the write instead. + let cache = cache(); + let key = key("https://example.com/length-mismatch"); + let mut metadata = metadata_for(b"12345"); + metadata.body_len = 999; + + let err = run(cache.put(&key, &metadata, b"12345".to_vec(), Duration::from_secs(60))) + .expect_err("a length mismatch must be refused"); + assert!( + matches!(err, TemplateCacheError::Backend { .. }), + "expected a backend error, got {err:?}" + ); + } +} diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index c035799e9..e44d46f77 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -29,6 +29,7 @@ glob = { workspace = true } hex = { workspace = true } hmac = { workspace = true } http = { workspace = true } +httpdate = { workspace = true } iab_gpp = { workspace = true } jose-jwk = { workspace = true } log = { workspace = true } diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 7c1303dd4..de968301c 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -1,5 +1,7 @@ use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; -use trusted_server_core::html_processor::{HtmlProcessorConfig, create_html_processor}; +use trusted_server_core::html_processor::{ + BodyCloseInjection, HtmlProcessorConfig, create_html_processor, +}; use trusted_server_core::integrations::IntegrationRegistry; use trusted_server_core::streaming_processor::StreamProcessor as _; @@ -13,6 +15,10 @@ fn make_config() -> HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + // The benchmark measures URL rewriting, not ad injection, and + // `ad_slots_script` is `None` here — matching the previous behaviour, + // which inferred no body-close work from that. + body_close: BodyCloseInjection::None, suppress_datadome_client_side_tag: false, } } diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index cd11e1c14..fca317440 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -16,6 +16,8 @@ use crate::settings::vec_from_seq_or_map; const MAX_DYNAMIC_GAM_UNIT_PATH_BYTES: usize = 100; const MAX_SECTION_BYTES: usize = 100; +const DEFAULT_TEMPLATE_CACHE_MAX_AGE_SECONDS: u32 = 60; +const MAX_TEMPLATE_CACHE_MAX_AGE_SECONDS: u32 = 86_400; /// A single parsed segment of a [`gam_unit_path`](CreativeOpportunitySlot::gam_unit_path) template. #[derive(Debug, Clone)] @@ -191,6 +193,36 @@ const fn is_default_enabled(value: &bool) -> bool { *value == default_enabled() } +/// How per-user ad state reaches the page. +/// +/// `Inline` is the shipped behaviour: the auction result is injected before +/// `` and the root document is therefore uncacheable. `Esi` stores a +/// request-neutral shared template and fills its per-request byte seam at the edge. +/// +/// Spike-only, for the #1009 ESI validation. Remove with the spike. +/// +/// # Why the template must be request-neutral +/// +/// Under `Esi` the template is shared across visitors, so +/// nothing whose *presence* depends on the request may appear in it — not merely +/// nothing whose *value* does. `tsjs.adSlots` is the trap: its content is derived +/// from config and path, but whether it is emitted at all is gated on consent, +/// bot classification, prefetch status and the auction kill switch. A template +/// filled by the first request would freeze that request's decision for every +/// later reader. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AssemblyMode { + /// Inject bids inline before ``. Root uncacheable. Shipped behaviour. + #[default] + Inline, + /// Serve a shared template; assemble its inert marker with an exact byte split. + /// + /// The operator-facing spelling remains `esi` for continuity, but no general + /// purpose ESI parser executes on this path. + Esi, +} + /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -261,11 +293,104 @@ pub struct CreativeOpportunitiesConfig { /// [`section_root`](Self::section_root) are omitted. #[serde(default, skip_serializing_if = "Option::is_none")] pub section_segment: Option, + /// How per-user ad state reaches the page. Absent means + /// [`AssemblyMode::Inline`], the shipped behaviour. + /// + /// `Option` rather than a bare enum, and `skip_serializing_if`, deliberately: + /// these structs use `deny_unknown_fields`, so a pushed key makes an older + /// binary fail configuration load. Keeping it absent when unset means a + /// deployment that never sets it stays rollback-compatible. + /// + /// Spike-only. See [`AssemblyMode`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub assembly_mode: Option, + /// Request headers the origin varies on, which the shared-template cache key must + /// cover. + /// + /// Operator-stated because a cache **lookup happens before the fetch**, so on a cold + /// key the origin's `Vary` is not yet known. See `VarySpec` for why the alternatives + /// (two-phase lookup, or storing the list and re-keying) were not taken. + /// + /// **Unset or empty means no operator-stated header is covered, so any origin + /// `Vary` other than structurally covered `Accept-Encoding` disqualifies the + /// response.** `Cookie` may never be configured: a per-cookie object violates the + /// reader-neutral template contract. This fail-closed default prevents a deployment + /// that has not stated what its origin varies on from gaining a shared cache by + /// omission. + /// + /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub template_cache_vary: Option>, + /// Maximum time a reader-neutral transformed template may remain in C2. + /// + /// This is a safety ceiling, not freshness authorization. The origin must still + /// provide positive shared freshness, and the stored lifetime is the smaller of + /// the origin's remaining edge freshness and this value. Defaults to 60 seconds + /// and may be configured from 1 second through 1 day. + /// + /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub template_cache_max_age_seconds: Option, + /// Operator assertion that the origin's HTML does not depend on request cookies. + /// + /// Unset or `false` disqualifies **every cookie-bearing request** from the shared + /// template cache, in both directions. That is safe and it is also very nearly a + /// disable switch: Trusted Server sets its own identity cookie, so essentially every + /// repeat visitor carries one. Left at the default, the cache can only ever serve + /// first-ever page views and cookie-less clients. + /// + /// Setting `true` asserts the origin serves the same HTML with or without cookies. + /// It is not taken on trust alone — if the origin ever declares `Vary: Cookie`, the + /// response is refused regardless of this flag or the configured key. So a wrong + /// assertion is caught whenever the origin is honest about it, and this only widens + /// the window where the origin personalizes *silently*. + /// + /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin_is_cookie_independent: Option, /// Slot templates. An empty vec or `enabled = false` disables template delivery. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub slot: Vec, } +impl CreativeOpportunitiesConfig { + /// Resolved assembly mode, defaulting to [`AssemblyMode::Inline`] when unset. + #[must_use] + pub fn assembly_mode(&self) -> AssemblyMode { + self.assembly_mode.unwrap_or_default() + } + + /// Whether a cookie-bearing request may participate in the shared cache. + /// + /// Defaults to `false`, which is the conservative reading and also the one that + /// makes the cache almost inert on real traffic. See + /// [`Self::origin_is_cookie_independent`]. + #[must_use] + pub fn origin_is_cookie_independent(&self) -> bool { + self.origin_is_cookie_independent.unwrap_or(false) + } + + /// Headers the cache key covers, per operator config. + /// + /// Unset yields an empty operator spec, so any origin `Vary` other than the + /// structurally covered `Accept-Encoding` reads as a gap and the response is never + /// cached. Failing closed is deliberate: an unconfigured deployment should not + /// acquire a shared cache silently. + #[must_use] + pub fn template_cache_vary(&self) -> crate::platform::VarySpec { + crate::platform::VarySpec::new(self.template_cache_vary.clone().unwrap_or_default()) + } + + /// Safety ceiling for one shared transformed-template cache entry. + #[must_use] + pub fn template_cache_max_age(&self) -> std::time::Duration { + std::time::Duration::from_secs(u64::from( + self.template_cache_max_age_seconds + .unwrap_or(DEFAULT_TEMPLATE_CACHE_MAX_AGE_SECONDS), + )) + } +} + impl CreativeOpportunitiesConfig { /// Derives the `{section}` value for `path` under this config's section /// policy ([`section_root`](Self::section_root) and @@ -333,10 +458,32 @@ impl CreativeOpportunitiesConfig { /// Returns an error string when [`gam_network_id`](Self::gam_network_id) is /// blank but consumed by a default path or `{network_id}` template; when a /// slot has an invalid identifier, page pattern set, format list, or - /// dimensions; when a `{section}` template lacks a valid + /// dimensions; when `template_cache_max_age_seconds` falls outside 1–86,400; + /// when a `{section}` template lacks a valid /// [`section_root`](Self::section_root); or when configured values make a /// dynamic path exceed 100 UTF-8 bytes. pub fn validate_runtime(&self) -> Result<(), String> { + if self + .template_cache_max_age_seconds + .is_some_and(|seconds| !(1..=MAX_TEMPLATE_CACHE_MAX_AGE_SECONDS).contains(&seconds)) + { + return Err(format!( + "template_cache_max_age_seconds must be between 1 and {MAX_TEMPLATE_CACHE_MAX_AGE_SECONDS}" + )); + } + + if let Some(names) = &self.template_cache_vary { + crate::platform::VarySpec::try_new(names.clone()).map_err(|name| { + format!("template_cache_vary contains invalid HTTP header name `{name}`") + })?; + if names.iter().any(|name| name.eq_ignore_ascii_case("cookie")) { + return Err( + "template_cache_vary must not include Cookie; C2 templates are reader-neutral" + .to_string(), + ); + } + } + // A network ID is required only when a slot renders the default // `//` path or substitutes `{network_id}`. Static // and `{slot_id}`/`{section}`-only templates leave it inert. @@ -1197,6 +1344,10 @@ mod tests { auction_timeout_ms: None, price_granularity: PriceGranularity::default(), section_root: section_root.map(str::to_string), + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, section_segment: None, slot: vec![slot], } @@ -1595,6 +1746,10 @@ mod tests { auction_timeout_ms: None, price_granularity: PriceGranularity::default(), section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, section_segment: None, slot: Vec::new(), }; @@ -1872,6 +2027,164 @@ mod tests { ); } + #[test] + fn assembly_mode_defaults_to_inline_when_absent() { + // Arrange: the minimal config an existing deployment would have. + let toml = r#" + gam_network_id = "99999" + "#; + + // Act + let config: CreativeOpportunitiesConfig = + toml::from_str(toml).expect("should deserialize without assembly_mode"); + + // Assert + assert_eq!( + config.assembly_mode, None, + "an absent key should stay absent rather than materializing a value" + ); + assert_eq!( + config.assembly_mode(), + AssemblyMode::Inline, + "should resolve to the shipped inline behaviour" + ); + } + + #[test] + fn assembly_mode_deserializes_each_variant() { + for (raw, expected) in [("inline", AssemblyMode::Inline), ("esi", AssemblyMode::Esi)] { + let toml = format!( + r#" + gam_network_id = "99999" + assembly_mode = "{raw}" + "# + ); + let config: CreativeOpportunitiesConfig = + toml::from_str(&toml).unwrap_or_else(|e| panic!("should parse {raw}: {e}")); + assert_eq!( + config.assembly_mode(), + expected, + "should resolve `{raw}` to {expected:?}" + ); + } + + let removed_mode = r#" + gam_network_id = "99999" + assembly_mode = "client_fill" + "#; + assert!( + toml::from_str::(removed_mode).is_err(), + "client_fill is outside #1009's ESI byte-seam design and must be rejected" + ); + } + + #[test] + fn template_cache_vary_rejects_invalid_header_names() { + let config: CreativeOpportunitiesConfig = toml::from_str( + r#" + gam_network_id = "99999" + template_cache_vary = ["rsc", "not a header"] + "#, + ) + .expect("shape should deserialize before runtime validation"); + let err = config + .validate_runtime() + .expect_err("invalid field names must fail configuration validation"); + assert!(err.contains("not a header"), "unexpected error: {err}"); + + let cookie_key: CreativeOpportunitiesConfig = toml::from_str( + r#" + gam_network_id = "99999" + template_cache_vary = ["Cookie"] + "#, + ) + .expect("shape should deserialize before runtime validation"); + let err = cookie_key + .validate_runtime() + .expect_err("per-cookie templates violate the reader-neutral C2 contract"); + assert!(err.contains("Cookie"), "unexpected error: {err}"); + } + + #[test] + fn template_cache_max_age_accepts_a_positive_value_up_to_one_day() { + for seconds in [1_u32, 1_200, 86_400] { + let config: CreativeOpportunitiesConfig = toml::from_str(&format!( + r#" + gam_network_id = "99999" + template_cache_max_age_seconds = {seconds} + "# + )) + .unwrap_or_else(|error| panic!("{seconds}s should deserialize: {error}")); + + config + .validate_runtime() + .unwrap_or_else(|error| panic!("{seconds}s should validate: {error}")); + let serialized = serde_json::to_value(config).expect("should serialize config"); + assert_eq!( + serialized + .get("template_cache_max_age_seconds") + .and_then(serde_json::Value::as_u64), + Some(u64::from(seconds)), + "the configured ceiling must survive typed configuration" + ); + } + } + + #[test] + fn template_cache_max_age_rejects_zero_and_more_than_one_day() { + for seconds in [0_u32, 86_401] { + let config: CreativeOpportunitiesConfig = toml::from_str(&format!( + r#" + gam_network_id = "99999" + template_cache_max_age_seconds = {seconds} + "# + )) + .unwrap_or_else(|error| panic!("shape should deserialize before validation: {error}")); + + let error = config + .validate_runtime() + .expect_err("an unsafe template-cache ceiling must fail startup validation"); + assert!( + error.contains("template_cache_max_age_seconds"), + "unexpected validation error: {error}" + ); + } + } + + #[test] + fn unset_template_cache_max_age_is_omitted_for_rollback_compatibility() { + let config: CreativeOpportunitiesConfig = + toml::from_str("gam_network_id = \"99999\"").expect("should deserialize"); + + assert_eq!( + config.template_cache_max_age(), + std::time::Duration::from_secs(60), + "an absent ceiling must preserve the spike's existing lifetime" + ); + let serialized = toml::to_string(&config).expect("should serialize"); + + assert!( + !serialized.contains("template_cache_max_age_seconds"), + "an unset new key must not break rollback to an older binary: {serialized}" + ); + } + + #[test] + fn unset_assembly_mode_is_omitted_from_serialized_config() { + // `deny_unknown_fields` means a pushed key breaks config load on an older + // binary. A deployment that never sets this must not gain the key just by + // round-tripping through a newer one. + let config: CreativeOpportunitiesConfig = + toml::from_str("gam_network_id = \"99999\"").expect("should deserialize"); + + let serialized = toml::to_string(&config).expect("should serialize"); + + assert!( + !serialized.contains("assembly_mode"), + "unset assembly_mode must not be serialized, got:\n{serialized}" + ); + } + #[test] fn prebid_slot_params_deserializes_without_bidders_field() { let json = r#"{"bidders": {}}"#; diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 3bff588fe..711c4e31d 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use lol_html::{ - EndTagHandler, Settings as RewriterSettings, element, + EndTagHandler, Settings as RewriterSettings, doc_comments, element, end, html_content::{ContentType, EndTag}, text, }; @@ -156,6 +156,29 @@ impl StreamProcessor for HtmlWithPostProcessing { fn reset(&mut self) {} } +/// What the `` seam injects. +/// +/// This is a decision, not a side effect of whether the `` script exists. +/// An earlier shape gated body-close injection on `ad_slots_script.is_some()`, +/// which coupled two independent choices: once a shared-template mode stopped +/// emitting the head script, body-close injection silently stopped too. +/// +/// See `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +/// §6.7. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum BodyCloseInjection { + /// Emit nothing because no slots matched under the inline path. + #[default] + None, + /// Read the auction result from `ad_bids_state` and inject it, falling back to + /// an empty payload. Today's shipped behaviour. + InlineBids, + /// Emit this markup verbatim — an inert marker the assembly step splits on. + /// Must be identical for every request that reaches the transform, or the + /// cached template is not shared-safe. + Marker(String), +} + /// Configuration for HTML processing #[derive(Clone)] pub struct HtmlProcessorConfig { @@ -176,6 +199,9 @@ pub struct HtmlProcessorConfig { pub max_buffered_body_bytes: usize, /// Request-scoped conditional diagnostics delivery decision. pub gpt_diagnostics: Option, + /// What the `` seam injects. Decided by the caller rather than inferred + /// from [`Self::ad_slots_script`]. + pub body_close: BodyCloseInjection, /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. pub suppress_datadome_client_side_tag: bool, } @@ -199,6 +225,7 @@ impl HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: settings.publisher.max_buffered_body_bytes, gpt_diagnostics: None, + body_close: BodyCloseInjection::None, suppress_datadome_client_side_tag: false, } } @@ -221,6 +248,17 @@ impl HtmlProcessorConfig { self } + /// Set what the `` seam injects. + /// + /// Separate from [`with_ad_state`](Self::with_ad_state) because the two are + /// independent decisions: a shared-template mode emits no head script and + /// still needs a body-close marker. + #[must_use] + pub fn with_body_close(mut self, body_close: BodyCloseInjection) -> Self { + self.body_close = body_close; + self + } + /// Attach the request-scoped conditional diagnostics decision. #[must_use] pub fn with_gpt_diagnostics(mut self, decision: Option) -> Self { @@ -318,9 +356,44 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let integration_registry = config.integrations.clone(); let script_rewriters = integration_registry.script_rewriters(); let ad_slots_script = config.ad_slots_script.clone(); + let body_close = config.body_close.clone(); let ad_bids_state = config.ad_bids_state.clone(); let gpt_diagnostics = config.gpt_diagnostics.clone(); + // A publisher can legitimately emit the same inert comment text as the reserved + // C2 seam, including after ``. Neutralize source comments while they are + // parsed; markup injected by the body end-tag handler is output, not reparsed, so + // the transform-owned marker remains the only exact copy. + let mut document_content_handlers = Vec::new(); + if let BodyCloseInjection::Marker(marker) = &body_close + && let Some(reserved) = marker + .strip_prefix("")) + { + let reserved = reserved.to_string(); + let escaped = format!("x{reserved}"); + document_content_handlers.push(doc_comments!(move |comment| { + if comment.text() == reserved { + comment.set_text(&escaped)?; + } + Ok(()) + })); + } + if let BodyCloseInjection::Marker(marker) = &body_close { + let marker = marker.clone(); + let injected_bids = Arc::clone(&injected_bids); + document_content_handlers.push(end!(move |document_end| { + // HTML fragments and malformed-but-renderable documents may never expose a + // body end tag. Always mint a transform-owned terminal seam in that case; + // otherwise source bytes equal to the reserved marker could be mistaken for + // ownership by the post-transform exact-count validator. + if !injected_bids.swap(true, Ordering::SeqCst) { + document_end.append(&marker, ContentType::Html); + } + Ok(()) + })); + } + let mut element_content_handlers = vec![ // Inject unified tsjs bundle once at the start of element!("head", { @@ -385,29 +458,42 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso element!("body", { let state = ad_bids_state.clone(); let injected_bids = injected_bids.clone(); - let has_slots = ad_slots_script.is_some(); + let body_close = body_close.clone(); move |el| { - if !has_slots { + if matches!(body_close, BodyCloseInjection::None) { return Ok(()); } let state = state.clone(); let injected_bids = injected_bids.clone(); + let body_close = body_close.clone(); if let Some(handlers) = el.end_tag_handlers() { let handler: EndTagHandler<'static> = Box::new(move |end_tag: &mut EndTag<'_>| { if injected_bids.swap(true, Ordering::SeqCst) { return Ok(()); } - let script_guard = state.lock().expect("should lock bid state"); - let bids_script = match &*script_guard { - Some(s) => s.clone(), - None => build_empty_bids_script(), + let markup = match &body_close { + // Verbatim, and identical on every request that + // reaches the transform — that is what makes the + // cached template shared-safe. + BodyCloseInjection::Marker(marker) => marker.clone(), + BodyCloseInjection::InlineBids => { + let script_guard = state.lock().expect("should lock bid state"); + match &*script_guard { + Some(s) => s.clone(), + None => build_empty_bids_script(), + } + } + // Unreachable: the element handler returned early + // above. Kept exhaustive rather than using `_` so a + // new variant is a compile error here. + BodyCloseInjection::None => return Ok(()), }; - end_tag.before(&bids_script, ContentType::Html); + end_tag.before(&markup, ContentType::Html); Ok(()) }); handlers.push(handler); - } else { + } else if matches!(body_close, BodyCloseInjection::InlineBids) { // No end tag (implicitly closed or EOF ``): lol_html // cannot attach an end-tag handler, so tsjs.bids/adInit() are // never injected even though adSlots was injected at ``. @@ -659,6 +745,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso } let rewriter_settings = RewriterSettings { + document_content_handlers, element_content_handlers, ..RewriterSettings::default() }; @@ -698,6 +785,7 @@ mod tests { fn create_test_config() -> HtmlProcessorConfig { HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_owned(), request_host: "test.example.com".to_owned(), request_scheme: "https".to_owned(), @@ -1599,6 +1687,7 @@ mod tests { #[test] fn injects_ad_slots_at_head_open() { let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1675,6 +1764,7 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1712,6 +1802,7 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1750,6 +1841,7 @@ mod tests { let request_host = "proxy.test-publisher.example.com"; let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.test-publisher.example.com".to_string(), request_host: request_host.to_string(), request_scheme: "https".to_string(), @@ -1802,6 +1894,7 @@ mod tests { // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1832,6 +1925,7 @@ mod tests { // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1853,6 +1947,41 @@ mod tests { ); } + #[test] + fn bodyless_marker_mode_emits_an_owned_terminal_seam_even_after_source_bytes() { + const MARKER: &str = ""; + let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::Marker(MARKER.to_string()), + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), + max_buffered_body_bytes: 16 * 1024 * 1024, + gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, + }; + let source = + format!(r#""#); + + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(source.as_bytes(), true) + .expect("should process bodyless HTML"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + + assert_eq!( + html.matches(MARKER).count(), + 2, + "one source occurrence plus one transform-owned seam must reach normalization" + ); + assert!( + html.ends_with(MARKER), + "the transform-owned fallback must be unambiguously terminal" + ); + } + #[test] fn response_size_does_not_grow_disproportionately() { // Processing must not expand HTML by more than 1.1× (accounts for the diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 665060014..75de88afb 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -58,14 +58,21 @@ impl DataDomeIntegration { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } + let request_method = input.request.method().clone(); match self.filter_protection_request_inner(input).await { Ok(decision) => decision, Err(ProtectionRequestError::Setup(err)) => { - log::error!("[datadome] Protection setup failed open: {err:?}"); + log::error!( + "[datadome] protection decision=failed_open api_status=none datadome_status=unavailable method={} route=continue failure=setup error={err:?}", + request_method, + ); RequestFilterDecision::Continue(RequestFilterEffects::default()) } Err(ProtectionRequestError::Runtime(err)) => { - log::warn!("[datadome] Protection API failed open: {err:?}"); + log::warn!( + "[datadome] protection decision=failed_open api_status=none datadome_status=unavailable method={} route=continue failure=runtime error={err:?}", + request_method, + ); RequestFilterDecision::Continue(RequestFilterEffects::default()) } } @@ -183,15 +190,15 @@ impl DataDomeIntegration { if supplied_values.is_empty() { return false; } + let Some(bypass) = self.active_protection_test_bypass() else { + return false; + }; if supplied_values.len() != 1 { log::warn!( "[datadome] Multiple DataDome test bypass headers supplied; ignoring bypass" ); return false; } - let Some(bypass) = self.active_protection_test_bypass() else { - return false; - }; let store_name = StoreName::from(bypass.credential_secret_store.as_str()); let credential = match services @@ -522,31 +529,21 @@ fn log_protection_skip( reason: ProtectionSkipReason, suppress_client_tag: bool, ) { - let reason = reason.as_str(); - if suppression_skip_log_level(suppress_client_tag, is_navigation_request(input.request)) - == log::Level::Info - { - log::info!( - "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", - rule_id, - reason, - input.request.method(), - ); - } else if suppress_client_tag { - log::debug!( - "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", - rule_id, - reason, - input.request.method(), - ); + let level = + suppression_skip_log_level(suppress_client_tag, is_navigation_request(input.request)); + let client_tag = if suppress_client_tag { + " client_tag=omitted" } else { - log::debug!( - "[datadome] protection decision=skipped rule={} reason={} method={}", - rule_id, - reason, - input.request.method(), - ); - } + "" + }; + log::log!( + level, + "[datadome] protection decision=skipped rule={} reason={}{} method={}", + rule_id, + reason.as_str(), + client_tag, + input.request.method(), + ); } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -839,7 +836,7 @@ fn truncate_utf8(value: &str, limit: i32) -> String { mod tests { use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr}; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; use crate::integrations::datadome::{ DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, @@ -855,6 +852,8 @@ mod tests { use super::*; + static FASTLY_IS_STAGING_ENV_LOCK: Mutex<()> = Mutex::new(()); + fn protection_integration() -> Arc { let config = DataDomeConfig { enabled: true, @@ -878,6 +877,9 @@ mod tests { services: &RuntimeServices, request: &mut Request, ) -> RequestFilterDecision { + let _guard = FASTLY_IS_STAGING_ENV_LOCK + .lock() + .expect("should lock staging environment test guard"); temp_env::with_var(crate::constants::ENV_FASTLY_IS_STAGING, Some("1"), || { futures::executor::block_on(integration.filter_protection_request(RequestFilterInput { settings, @@ -1098,6 +1100,9 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); + let _guard = FASTLY_IS_STAGING_ENV_LOCK + .lock() + .expect("should lock staging environment test guard"); let decision = temp_env::with_var( crate::constants::ENV_FASTLY_IS_STAGING, None::<&str>, diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 86b51ffa7..43934771d 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -94,8 +94,12 @@ // and deliberately identical to the bundle scheduler — the impression is // spent on a viewed tab, and the post-hydration guarantee holds whenever // the request is actually issued. - ts.scheduleInitialAdInit = function (initialBids) { + ts.scheduleInitialAdInit = function (initialBids, initialSlots) { if ((ts.navGeneration || 0) !== 0) return; + // Slots are generation-guarded for the same reason the bids are: the + // shared-template seam sends both, and an assignment made before this call + // would overwrite a committed SPA navigation's slots. + if (initialSlots) ts.adSlots = initialSlots; if (initialBids) ts.bids = initialBids; var fire = function () { if ((ts.navGeneration || 0) !== 0) return; diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 5bd86d19e..7a91eead4 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -114,6 +114,83 @@ impl GptDiagnosticsRequestDecision { } } +impl GptDiagnosticsRequestDecision { + /// An active decision, for tests in other modules that need one. + /// + /// The fields are private and built by `prepare_request` from a cookie or query + /// parameter; there is no other way to obtain an active decision across a module + /// boundary. + #[cfg(test)] + #[must_use] + pub(crate) fn active_for_tests() -> Self { + Self { + active: true, + clean_browser_path_and_query: None, + cookie_action: GptDiagnosticsCookieAction::None, + } + } +} + +#[cfg(test)] +mod head_seam_invariant_tests { + use super::*; + + /// Every combination of the three fields the decision carries. + fn all_decisions() -> Vec { + let mut out = Vec::new(); + for active in [false, true] { + for clean in [None, Some("/clean".to_string())] { + for cookie_action in [ + GptDiagnosticsCookieAction::None, + GptDiagnosticsCookieAction::SetSession, + GptDiagnosticsCookieAction::ClearSession, + ] { + out.push(GptDiagnosticsRequestDecision { + active, + clean_browser_path_and_query: clean.clone(), + cookie_action, + }); + } + } + } + out + } + + #[test] + fn requires_private_no_store_is_a_superset_of_injection() { + // Load-bearing relationship, not an incidental one. Whenever this decision + // injects anything into ``, the response must also be stamped + // `private, no-store` — which is what keeps request-scoped diagnostics out + // of a shared cache if the explicit assembly-mode gate in + // `create_html_stream_processor` is ever removed or bypassed. + // + // If a future change makes a script emit without also requiring the stamp, + // this fails here rather than silently in a cached template. + for decision in all_decisions() { + let injects = + decision.bootstrap_script().is_some() || decision.module_script_tag().is_some(); + if injects { + assert!( + decision.requires_private_no_store(), + "decision injects into but does not require private/no-store: \ + {decision:?}" + ); + } + } + } + + #[test] + fn a_default_decision_injects_nothing() { + let decision = GptDiagnosticsRequestDecision::default(); + assert_eq!(decision.bootstrap_script(), None); + assert_eq!(decision.module_script_tag(), None); + assert!( + !decision.requires_private_no_store(), + "an inert decision should not force the response private" + ); + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum QueryDirective { Absent, diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 287f1accf..7c80f9e12 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -12,6 +12,7 @@ //! - [`PlatformBackend`] — dynamic backend registration //! - [`PlatformHttpClient`] — outbound HTTP client //! - [`PlatformGeo`] — geographic information lookup +//! - [`PlatformTemplateAssembler`] — cold-response shared-template assembly //! //! ## Platform-Agnostic Components //! @@ -36,6 +37,8 @@ mod error; mod http; mod image_optimizer; mod kv; +mod template_assembly; +mod template_cache; #[cfg(test)] pub(crate) mod test_support; mod traits; @@ -52,6 +55,16 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; +pub use template_assembly::{ + PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, +}; +pub use template_cache::REPLAYABLE_POLICY_HEADERS; +pub use template_cache::{ + PlatformTemplateCache, PlatformTemplateCacheReservation, TEMPLATE_SCHEMA_VERSION, + TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, + TemplateCacheReservation, TemplateEntry, TemplateMetadata, UnavailableTemplateCache, + VaryHeaderValues, VarySpec, +}; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, diff --git a/crates/trusted-server-core/src/platform/template_assembly.rs b/crates/trusted-server-core/src/platform/template_assembly.rs new file mode 100644 index 000000000..441e7ca31 --- /dev/null +++ b/crates/trusted-server-core/src/platform/template_assembly.rs @@ -0,0 +1,77 @@ +//! Platform boundary for assembling a shared template with reader-specific state. +//! +//! Core owns the cache-safety ordering and the portable byte-seam fallback. An adapter +//! may provide a richer assembler for the cold response after the reader-neutral +//! template has been stored. + +use core::fmt; + +/// Why a platform assembler could not produce a document. +#[derive(Debug, derive_more::Display)] +pub enum TemplateAssemblyError { + /// The adapter has no template assembler. + #[display("this adapter cannot assemble shared templates")] + Unsupported, + /// The platform assembler rejected or could not process the document. + #[display("template assembly failed: {message}")] + Failed { + /// Human-readable failure context. + message: String, + }, +} + +impl core::error::Error for TemplateAssemblyError {} + +/// Assembles reader-specific state into a shared HTML template. +pub trait PlatformTemplateAssembler: Send + Sync { + /// Produce the complete document served to this reader. + /// + /// # Errors + /// + /// Returns [`TemplateAssemblyError`] when the adapter cannot assemble the template. + fn assemble(&self, template: &[u8], fragment: &[u8]) -> Result, TemplateAssemblyError>; +} + +impl fmt::Debug for dyn PlatformTemplateAssembler { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformTemplateAssembler") + } +} + +/// Default assembler used by adapters that do not provide platform assembly. +#[derive(Debug, Default, Clone, Copy)] +pub struct UnavailableTemplateAssembler; + +impl PlatformTemplateAssembler for UnavailableTemplateAssembler { + fn assemble( + &self, + _template: &[u8], + _fragment: &[u8], + ) -> Result, TemplateAssemblyError> { + Err(TemplateAssemblyError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unavailable_assembler_refuses_the_document() { + let error = UnavailableTemplateAssembler + .assemble(b"", b"") + .expect_err("should refuse when platform assembly is unavailable"); + + assert!(matches!(error, TemplateAssemblyError::Unsupported)); + } + + #[test] + fn assembler_contract_is_object_safe() { + let assembler: Box = Box::new(UnavailableTemplateAssembler); + + assert!(matches!( + assembler.assemble(b"template", b"fragment"), + Err(TemplateAssemblyError::Unsupported) + )); + } +} diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs new file mode 100644 index 000000000..ac3761b0f --- /dev/null +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -0,0 +1,1096 @@ +//! The shared transformed-template cache (C2) for the #1009 ESI validation spike. +//! +//! Three caches are in play and conflating them is what produced the original wrong +//! conclusion in the design doc, so this module names which one it is: +//! +//! | Cache | Contents | Owner | +//! | ----- | --------------------------------- | ------------------------------ | +//! | C1 | raw origin bytes | Fastly read-through. Not this. | +//! | C2 | post-`lol_html`, pre-assembly | **This module.** | +//! | C3 | final per-user assembled response | **Must never exist.** | +//! +//! C2 holds a *shared template*: no per-user bytes, and no decisions that depend on +//! the request. What may and may not live in it is +//! [§6.7 of the design doc](../../../../docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md), +//! and the invariant is enforced by the rendered-document byte-identity tests in +//! `publisher`. +//! +//! Spike-only. Remove with the spike. + +use core::fmt; +use std::collections::HashSet; + +use crate::creative_opportunities::AssemblyMode; + +/// Version of the transform that produced a cached template. +/// +/// Bump on **any** change to what the transform emits. Without it a deploy reads +/// yesterday's template shape and assembles against markers that moved, which fails +/// as a rendering bug far from its cause rather than as a cache miss. +/// +/// | Version | Transform | +/// | ------- | --------- | +/// | 1 | `` seam used an executable ESI include tag targeting the old fragment endpoint | +/// | 2 | Marker became the inert comment ``; the seam hands slots to `scheduleInitialAdInit` instead of assigning them | +/// | 3 | Marker became ``; canonical collision-safe key, explicit origin freshness, and complete repeated document-policy metadata | +/// | 4 | Marker is the shorter, accurate [`AD_ASSEMBLY_SEAM`](crate::publisher::AD_ASSEMBLY_SEAM) | +pub const TEMPLATE_SCHEMA_VERSION: u32 = 4; + +/// Inputs that select one cached template. +/// +/// Every field changes the emitted bytes for the same URL. A signal that changes the +/// bytes and is **not** here produces cross-served templates; a signal that is +/// per-user does not belong here at all — it belongs out of the template entirely. +/// That distinction is the whole design: the key holds per-*variant* signals, and +/// per-*user* signals are excluded from the template rather than keyed on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateCacheKey { + /// Full request URL, stated explicitly rather than inherited from an ambient + /// request, so the key cannot silently depend on what the caller happened to + /// mutate first. + pub url: String, + /// Host and scheme. The post-processed output is host-dependent by construction: + /// both reach `IntegrationHtmlContext` and drive URL rewriting. + pub request_host: String, + /// See [`Self::request_host`]. + pub request_scheme: String, + /// Publisher origin identity, including the outbound Host override. Two virtual + /// hosts can share a connection target while producing unrelated documents. + pub origin_identity: String, + /// Inline and ESI modes emit different template bytes. Without this they poison + /// each other's entries. + pub assembly_mode: AssemblyMode, + /// Values of the request headers the **origin** declares it varies on, in the + /// order the origin listed them. Not a fixed list: the origin is authoritative, + /// and hard-coding one here would silently drift when the origin's changes. + pub vary_values: Vec, + /// Digest of every setting that can shape the transformed template plus the tsjs + /// bundle. Over-invalidating is safe; omitting a shaping input cross-serves bytes. + pub template_fingerprint: String, + /// See [`TEMPLATE_SCHEMA_VERSION`]. + pub schema_version: u32, +} + +impl TemplateCacheKey { + /// Render a fixed-size opaque key for the platform cache. + /// + /// The canonical input is length-prefixed before hashing, so neither delimiters nor + /// raw request values can collide or leak into cache diagnostics. + #[must_use] + pub fn to_cache_key(&self) -> String { + use sha2::Digest as _; + + fn push(out: &mut Vec, part: &[u8]) { + out.extend_from_slice(&(part.len() as u64).to_be_bytes()); + out.extend_from_slice(part); + } + + let mut canonical = Vec::new(); + push(&mut canonical, b"ts-c2"); + push(&mut canonical, &self.schema_version.to_be_bytes()); + push( + &mut canonical, + match self.assembly_mode { + AssemblyMode::Inline => b"inline", + AssemblyMode::Esi => b"esi", + }, + ); + push(&mut canonical, self.request_scheme.as_bytes()); + push(&mut canonical, self.request_host.as_bytes()); + push(&mut canonical, self.origin_identity.as_bytes()); + push(&mut canonical, self.url.as_bytes()); + push(&mut canonical, self.template_fingerprint.as_bytes()); + push( + &mut canonical, + &(self.vary_values.len() as u64).to_be_bytes(), + ); + for varied in &self.vary_values { + push(&mut canonical, varied.name.as_bytes()); + match &varied.values { + None => push(&mut canonical, b"absent"), + Some(values) => { + push(&mut canonical, b"present"); + push(&mut canonical, &(values.len() as u64).to_be_bytes()); + for value in values { + push(&mut canonical, value); + } + } + } + } + + let digest = sha2::Sha256::digest(canonical); + format!("ts-c2-v{}-{}", self.schema_version, hex::encode(digest)) + } + + /// Surrogate keys to attach at insert, for purge-based rollback. + /// + /// `ts-template` purges every template at once, which is the rollback lever. + /// The per-URL key allows targeted invalidation. Both are needed: the broad one + /// for an incident, the narrow one for ordinary invalidation. + #[must_use] + pub fn surrogate_keys(&self) -> Vec { + vec!["ts-template".to_string(), self.url_surrogate_key()] + } + + /// Surrogate key for every variant of this publisher URL. + /// + /// Used to evict a malformed object without flushing unrelated article templates. + #[must_use] + pub fn url_surrogate_key(&self) -> String { + format!("ts-template-url-{}", digest_hex(self.url.as_bytes())) + } +} + +fn digest_hex(bytes: &[u8]) -> String { + use sha2::Digest as _; + hex::encode(sha2::Sha256::digest(bytes)) +} + +/// One configured `Vary` input exactly as it appeared on the request. +/// +/// `None` means absent. `Some(vec![vec![]])` means present with one empty field +/// value. Repeated fields stay separate and ordered; no UTF-8 conversion is involved. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VaryHeaderValues { + /// Validated, lowercase header name. + pub name: String, + /// Every raw field value in wire order, or `None` when absent. + pub values: Option>>, +} + +/// Origin response headers safe to store with a shared template and replay on a hit. +/// +/// Every one is a per-URL policy statement, identical for every reader. Nothing +/// per-reader (`Set-Cookie`) and nothing cache-controlling (`Cache-Control`, `ETag`, +/// `Surrogate-Control`) appears here, and it is an allowlist so a new origin header is +/// excluded until someone decides otherwise. +pub const REPLAYABLE_POLICY_HEADERS: &[&str] = &[ + "content-security-policy", + "content-security-policy-report-only", + "permissions-policy", + "referrer-policy", + "strict-transport-security", + "cross-origin-opener-policy", + "cross-origin-embedder-policy", + "cross-origin-resource-policy", + "origin-agent-cluster", + "reporting-endpoints", + "report-to", + "link", + "x-frame-options", + "x-content-type-options", + "content-language", + "x-robots-tag", +]; + +/// Headers the key covers by construction, whatever the operator configured. +/// +/// The shared path stores decoded identity bytes and negotiates the reader representation +/// only after assembly, so an origin declaring `Vary: Accept-Encoding` is covered without +/// reader input. This assumes those origin variants differ only by HTTP content coding; +/// operators must leave ESI disabled if an origin changes document semantics instead. +/// Without this carve-out, the ordinary declaration sent by any compressing origin reads +/// as an uncovered gap and disqualifies the response, so **C2 would never cache anything +/// against a real origin** unless the operator redundantly listed a header the transform +/// already normalizes. Found by review before it could make the spike measure a hit rate +/// of approximately zero and read that as a result. +const STRUCTURALLY_COVERED: &[&str] = &["accept-encoding"]; + +/// Request headers to include in the cache key, and where the list comes from. +/// +/// # The chicken-and-egg this resolves +/// +/// The key must cover everything the origin varies on, or two requests needing +/// different templates share one entry. But a **lookup happens before the fetch**, +/// so on a cold key the origin's `Vary` is not yet known. +/// +/// Three ways out, and the trade-off is real: +/// +/// 1. **Configure the list** — what this does. One lookup, no extra round trip, and +/// the operator states what the origin varies on. Cost: it drifts silently if the +/// origin's `Vary` changes and nobody updates config. +/// 2. **Two-phase lookup** — fetch a URL-keyed record holding the last-seen `Vary`, +/// then key properly. Correct, but doubles the lookups on every request. +/// 3. **Store the list alongside** and re-key on mismatch. Same cost as (2) plus +/// complexity. +/// +/// (1) is chosen for the spike because Step A already measured the origin's actual +/// `Vary`, the origin response is checked for drift before storage, and the configured +/// template-cache ceiling bounds how long a newly introduced mismatch can survive. +/// **This is a spike-grade choice, not a production one** — see the drift guard below. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VarySpec { + /// Header names, lowercased, in a fixed order. + names: Vec, +} + +impl VarySpec { + /// Build from configured header names. + /// + /// # Panics + /// + /// Panics when a name is not a valid HTTP field name. Runtime configuration is + /// validated with [`Self::try_new`] before this constructor is used. + #[must_use] + pub fn new(names: impl IntoIterator) -> Self { + Self::try_new(names).expect("VarySpec names should be validated at configuration load") + } + + /// Build from configured names, validating and deduplicating them. + /// + /// # Errors + /// + /// Returns the offending name when it is not a valid HTTP field name. + pub fn try_new(names: impl IntoIterator) -> Result { + let mut seen = HashSet::new(); + let mut normalized = Vec::new(); + for raw in names { + let name = http::header::HeaderName::from_bytes(raw.as_bytes()) + .map_err(|_| raw.clone())? + .as_str() + .to_string(); + if STRUCTURALLY_COVERED.contains(&name.as_str()) { + continue; + } + if seen.insert(name.clone()) { + normalized.push(name); + } + } + Ok(Self { names: normalized }) + } + + /// Configured names, lowercased. + #[must_use] + pub fn names(&self) -> &[String] { + &self.names + } + + /// Extract the key inputs from a request's headers. + /// + /// A header the origin varies on but the request omits still contributes an + /// entry, with an empty value — otherwise "absent" and "present but empty" + /// would collide, and those are different requests to the origin. + #[must_use] + pub fn values_from(&self, headers: &http::HeaderMap) -> Vec { + self.names + .iter() + .map(|name| { + let values = headers.contains_key(name.as_str()).then(|| { + headers + .get_all(name.as_str()) + .iter() + .map(|value| value.as_bytes().to_vec()) + .collect() + }); + VaryHeaderValues { + name: name.clone(), + values, + } + }) + .collect() + } + + /// Whether the origin's declared `Vary` contains anything this spec omits. + /// + /// The drift guard for choice (1) above. Called **after** the origin responds, + /// when its `Vary` is finally known: if the origin varies on something the key + /// did not cover, the template just built is unsafe to store, because a request + /// differing only in that header would read it. + /// + /// Returns the uncovered names, so the caller can log precisely which config is + /// stale rather than reporting a generic refusal. + #[must_use] + pub fn uncovered_by<'a>(&self, origin_vary: impl IntoIterator) -> Vec { + origin_vary + .into_iter() + .flat_map(|value| value.split(',')) + .map(|name| name.trim().to_ascii_lowercase()) + .filter(|name| !name.is_empty() && name != "*") + .filter(|name| !STRUCTURALLY_COVERED.contains(&name.as_str())) + .filter(|name| !self.names.contains(name)) + .collect() + } +} + +/// Metadata stored alongside the template bytes. +/// +/// `cache::core` carries **no HTTP semantics** — status, headers, encoding and +/// revalidation are all the caller's. Rather than storing origin headers and +/// replaying them, store only what is needed to rebuild a response from scratch. +/// +/// That choice is deliberate and load-bearing: the publisher path forces +/// `private, no-store` and strips validators *after* the origin send, so replaying a +/// stored origin header would fight it. Rebuilding every header on a hit means no +/// origin header is ever replayed and the `Set-Cookie` privacy net stays trivially +/// safe. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateMetadata { + /// Encoding of the stored bytes. C2 writes only `identity`; retaining the field in + /// metadata makes corrupt or stale representations fail validation on read. + pub content_encoding: String, + /// Content type to rebuild the response with. + pub content_type: String, + /// Schema version the bytes were produced under. Checked on read: a mismatch is + /// a miss, not an error, so a rollback to an older binary degrades to + /// re-transforming rather than misassembling. + pub schema_version: u32, + /// Length of the template bytes as written. + /// + /// Guards against a partially written entry. `Transaction::insert` consumes the + /// transaction, so a write that fails part-way cannot cancel the insert — there + /// is no handle left to cancel it with. Recording the intended length and + /// checking it on read makes a truncated entry a miss instead of a silently + /// short template that would assemble into a broken page. + pub body_len: u64, + /// Origin response headers that are policy, not per-reader state. + /// + /// Reconstructing headers from scratch on a hit keeps origin `Set-Cookie` and caching + /// directives out of a shared cache — but it also dropped `Content-Security-Policy`, + /// framing protection and `Content-Language`, weakening the page. These are + /// per-URL and identical for every reader, so they belong with the template. + /// + /// Deliberately an allowlist: anything per-reader or cache-controlling is excluded by + /// construction rather than by remembering to strip it. + pub policy_headers: Vec<(String, String)>, +} + +impl TemplateMetadata { + /// Serialize for `user_metadata`. Deliberately a tiny hand-rolled format rather + /// than JSON — one allocation, no dependency, and a parse failure is + /// unambiguous. + #[must_use] + pub fn encode(&self) -> Vec { + let mut out = format!( + "v={}\nce={}\nct={}\nlen={}", + self.schema_version, self.content_encoding, self.content_type, self.body_len + ); + for (name, value) in &self.policy_headers { + // Header values cannot contain newlines (the HTTP parser rejects them), so a + // newline-delimited encoding cannot be broken by a header value. + out.push_str(&format!("\nh={name}:{value}")); + } + out.into_bytes() + } + + /// Parse `user_metadata`. Returns `None` on anything unexpected, which callers + /// must treat as a cache miss. + #[must_use] + pub fn decode(raw: &[u8]) -> Option { + let text = core::str::from_utf8(raw).ok()?; + let mut schema_version = None; + let mut policy_headers = Vec::new(); + let mut content_encoding = None; + let mut content_type = None; + let mut body_len = None; + for line in text.lines() { + let (key, value) = line.split_once('=')?; + match key { + "v" => { + if schema_version.replace(value.parse().ok()?).is_some() { + return None; + } + } + "ce" => { + if content_encoding.replace(value.to_string()).is_some() { + return None; + } + } + "h" => { + let (name, header_value) = value.split_once(':')?; + let name = http::header::HeaderName::from_bytes(name.as_bytes()).ok()?; + if !REPLAYABLE_POLICY_HEADERS.contains(&name.as_str()) { + return None; + } + http::HeaderValue::from_bytes(header_value.as_bytes()).ok()?; + policy_headers.push((name.as_str().to_string(), header_value.to_string())); + } + "ct" => { + if content_type.replace(value.to_string()).is_some() { + return None; + } + } + "len" => { + if body_len.replace(value.parse().ok()?).is_some() { + return None; + } + } + _ => return None, + } + } + let content_encoding = content_encoding?; + // Every template is decoded before insert. Accepting another value here would + // let corrupt metadata label plaintext bytes as gzip on a warm hit. + if content_encoding != "identity" { + return None; + } + let content_type = content_type?; + http::HeaderValue::from_bytes(content_type.as_bytes()).ok()?; + if !content_type + .split(';') + .next() + .is_some_and(|media_type| media_type.trim().eq_ignore_ascii_case("text/html")) + { + return None; + } + Some(Self { + schema_version: schema_version?, + policy_headers, + content_encoding, + content_type, + body_len: body_len?, + }) + } +} + +/// Why a template read did not produce usable bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum TemplateCacheMiss { + /// No entry for this key. + #[display("no cached template for this key")] + NotFound, + /// Found, but produced by a different transform version. + #[display("cached template has a different schema version")] + SchemaMismatch, + /// Found, but its metadata could not be parsed. + #[display("cached template metadata is unreadable")] + UnreadableMetadata, + /// Found, but shorter than the metadata says it should be — a write that failed + /// part-way. See [`TemplateMetadata::body_len`]. + #[display("cached template is truncated")] + Truncated, + /// This platform has no template cache. + #[display("no template cache on this platform")] + Unsupported, +} + +impl core::error::Error for TemplateCacheMiss {} + +/// Errors a template cache write can produce. +#[derive(Debug, derive_more::Display)] +pub enum TemplateCacheError { + /// This platform has no template cache. + #[display("no template cache on this platform")] + Unsupported, + /// The platform rejected the operation. + #[display("template cache backend error: {message}")] + Backend { + /// What the backend reported. + message: String, + }, +} + +impl core::error::Error for TemplateCacheError {} + +/// Result of the pre-origin cache transaction. +pub enum TemplateCacheLookup { + /// A fresh usable template. + Hit(TemplateEntry), + /// This request owns the obligation to provide or cancel the cold object. + Reserved(TemplateCacheReservation), + /// This adapter deliberately has no shared-template cache. + Unsupported, + /// A cache object existed but failed schema, metadata, or length validation. + Invalid(TemplateCacheMiss), +} + +/// Platform-owned insert obligation. Dropping it cancels, making every early-return +/// path safe without an async cleanup ladder in the publisher pipeline. +pub struct TemplateCacheReservation { + inner: Option>, +} + +impl core::fmt::Debug for TemplateCacheReservation { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TemplateCacheReservation") + .finish_non_exhaustive() + } +} + +impl TemplateCacheReservation { + /// Wrap a platform reservation. + #[must_use] + pub fn new(inner: Box) -> Self { + Self { inner: Some(inner) } + } + + /// Fulfil the reservation with a validated template. + /// + /// # Errors + /// + /// Returns the platform cache error when the reservation cannot be fulfilled. + pub fn insert( + mut self, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + self.inner + .take() + .ok_or_else(|| TemplateCacheError::Backend { + message: "template reservation was already consumed".to_string(), + })? + .insert(metadata, body, max_age) + } + + /// Explicitly give up the reservation. Drop performs the same operation as a net. + /// + /// # Errors + /// + /// Returns the platform cache error when the reservation cannot be cancelled. + pub fn cancel(mut self) -> Result<(), TemplateCacheError> { + self.inner + .take() + .ok_or_else(|| TemplateCacheError::Backend { + message: "template reservation was already consumed".to_string(), + })? + .cancel() + } +} + +impl Drop for TemplateCacheReservation { + fn drop(&mut self) { + if let Some(inner) = self.inner.take() + && let Err(err) = inner.cancel() + { + log::warn!("c2_template_cache reservation cancellation failed: {err}"); + } + } +} + +/// Adapter-specific ownership token returned by a transactional lookup. +pub trait PlatformTemplateCacheReservation: Send { + /// Insert and discharge the obligation. + /// + /// # Errors + /// + /// Returns an adapter-specific cache error when the insert fails. + fn insert( + self: Box, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError>; + + /// Cancel and allow a waiting request to take ownership. + /// + /// # Errors + /// + /// Returns an adapter-specific cache error when cancellation fails. + fn cancel(self: Box) -> Result<(), TemplateCacheError>; +} + +impl fmt::Debug for dyn PlatformTemplateCache { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformTemplateCache") + } +} + +/// A platform's shared-template cache. +/// +/// Only the Fastly adapter implements this; every other adapter uses +/// [`UnavailableTemplateCache`], which reports [`TemplateCacheMiss::Unsupported`] so +/// the caller transforms every time rather than failing. +/// +/// `Send + Sync` on the trait, `?Send` on the futures: `RuntimeServices` is held in a +/// `LazyLock` static, so the trait object must cross threads even though the futures +/// themselves never do — the platform layer is `!Send` by construction. +#[async_trait::async_trait(?Send)] +pub trait PlatformTemplateCache: Send + Sync { + /// Transactionally look up a template before origin work begins. + async fn lookup_or_reserve( + &self, + key: &TemplateCacheKey, + ) -> Result { + Ok(match self.get(key).await { + Ok(entry) => TemplateCacheLookup::Hit(entry), + Err(TemplateCacheMiss::Unsupported | TemplateCacheMiss::NotFound) => { + TemplateCacheLookup::Unsupported + } + Err(miss) => TemplateCacheLookup::Invalid(miss), + }) + } + + /// Read a template. `Err` is a miss, not a failure — every variant means + /// "transform it yourself". + async fn get(&self, key: &TemplateCacheKey) -> Result; + + /// Store a template. + /// + /// Callers must not call this without having consulted the C2 eligibility gate + /// first: this method stores what it is given and cannot tell a shared template + /// from a per-user one. + async fn put( + &self, + key: &TemplateCacheKey, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError>; + + /// Purge every cached variant for one publisher URL. + async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError>; + + /// Purge every stored template. The rollback lever. + async fn purge_all(&self) -> Result<(), TemplateCacheError>; +} + +/// A template read from the cache. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateEntry { + /// Metadata stored at insert. + pub metadata: TemplateMetadata, + /// The transformed template bytes. + pub body: Vec, +} + +/// The null object, used by every adapter without a template cache. +/// +/// Reporting [`TemplateCacheMiss::Unsupported`] rather than erroring means the +/// ESI assembly mode degrades to transforming per request on Cloudflare, Axum and Spin +/// instead of failing — the mode stays portable, only the caching is not. +pub struct UnavailableTemplateCache; + +#[async_trait::async_trait(?Send)] +impl PlatformTemplateCache for UnavailableTemplateCache { + async fn lookup_or_reserve( + &self, + _key: &TemplateCacheKey, + ) -> Result { + Ok(TemplateCacheLookup::Unsupported) + } + + async fn get(&self, _key: &TemplateCacheKey) -> Result { + Err(TemplateCacheMiss::Unsupported) + } + + async fn put( + &self, + _key: &TemplateCacheKey, + _metadata: &TemplateMetadata, + _body: Vec, + _max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } + + async fn purge_url(&self, _key: &TemplateCacheKey) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } + + async fn purge_all(&self) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn key() -> TemplateCacheKey { + TemplateCacheKey { + url: "https://example.com/news/article".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + origin_identity: "https://origin.example.com\0origin.example.com".to_string(), + assembly_mode: AssemblyMode::Esi, + vary_values: vec![VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }], + template_fingerprint: "abc123".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + } + } + + struct CountingReservation(Arc); + + impl PlatformTemplateCacheReservation for CountingReservation { + fn insert( + self: Box, + _metadata: &TemplateMetadata, + _body: Vec, + _max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + Ok(()) + } + + fn cancel(self: Box) -> Result<(), TemplateCacheError> { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + + #[test] + fn dropping_an_unfulfilled_reservation_cancels_exactly_once() { + let cancellations = Arc::new(AtomicUsize::new(0)); + drop(TemplateCacheReservation::new(Box::new( + CountingReservation(Arc::clone(&cancellations)), + ))); + assert_eq!(cancellations.load(Ordering::SeqCst), 1); + } + + /// Every field must change the key. A field that does not is a cross-serving + /// bug: two requests needing different templates would share one entry. + #[test] + fn every_field_changes_the_key() { + let base = key().to_cache_key(); + + let mut mode = key(); + mode.assembly_mode = AssemblyMode::Inline; + assert_ne!( + mode.to_cache_key(), + base, + "assembly mode must change the key" + ); + + let mut url = key(); + url.url = "https://example.com/other".to_string(); + assert_ne!(url.to_cache_key(), base, "url must change the key"); + + let mut host = key(); + host.request_host = "other.example.com".to_string(); + assert_ne!(host.to_cache_key(), base, "host must change the key"); + + let mut scheme = key(); + scheme.request_scheme = "http".to_string(); + assert_ne!(scheme.to_cache_key(), base, "scheme must change the key"); + + let mut origin = key(); + origin.origin_identity = "https://origin.example.com\0other.example.com".to_string(); + assert_ne!( + origin.to_cache_key(), + base, + "origin Host identity must change the key" + ); + + let mut fingerprint = key(); + fingerprint.template_fingerprint = "def456".to_string(); + assert_ne!( + fingerprint.to_cache_key(), + base, + "template fingerprint must change the key" + ); + + let mut schema = key(); + schema.schema_version = TEMPLATE_SCHEMA_VERSION + 1; + assert_ne!( + schema.to_cache_key(), + base, + "schema version must change the key" + ); + + let mut vary = key(); + vary.vary_values = vec![VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"0".to_vec()]), + }]; + assert_ne!(vary.to_cache_key(), base, "vary values must change the key"); + } + + /// The reason for length prefixes rather than a delimiter. + #[test] + fn values_containing_delimiters_cannot_collide() { + let mut a = key(); + a.request_host = "a".to_string(); + a.url = "b:c".to_string(); + + let mut b = key(); + b.request_host = "a:b".to_string(); + b.url = "c".to_string(); + + assert_ne!( + a.to_cache_key(), + b.to_cache_key(), + "field values containing the delimiter must not produce the same key; a \ + collision here serves one visitor's template to another" + ); + } + + #[test] + fn rendered_key_is_fixed_size_and_contains_no_request_material() { + let rendered = key().to_cache_key(); + assert_eq!(rendered.len(), "ts-c2-v3-".len() + 64); + for sensitive in ["example.com", "/news/article", "rsc", "abc123"] { + assert!( + !rendered.contains(sensitive), + "key leaked `{sensitive}`: {rendered}" + ); + } + } + + #[test] + fn vary_header_names_are_matched_case_insensitively() { + let mut upper = key(); + upper.vary_values = vec![VaryHeaderValues { + name: "RSC".to_ascii_lowercase(), + values: Some(vec![b"1".to_vec()]), + }]; + assert_eq!( + upper.to_cache_key(), + key().to_cache_key(), + "header names are case-insensitive, so casing must not split the cache" + ); + } + + #[test] + fn vary_values_are_order_sensitive() { + // The origin lists them in a fixed order and the caller preserves it, so a + // differing order means differing inputs rather than the same request. + let mut a = key(); + a.vary_values = vec![ + VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }, + VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"article".to_vec()]), + }, + ]; + let mut b = key(); + b.vary_values = vec![ + VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"article".to_vec()]), + }, + VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }, + ]; + assert_ne!(a.to_cache_key(), b.to_cache_key()); + } + + #[test] + fn surrogate_keys_carry_a_global_and_a_per_url_lever() { + let keys = key().surrogate_keys(); + assert!( + keys.contains(&"ts-template".to_string()), + "a global purge lever is what makes rollback possible" + ); + assert_eq!(keys.len(), 2, "global plus per-URL"); + assert!( + !keys[1].contains(char::is_whitespace), + "surrogate keys are space-delimited; whitespace would purge more than \ + intended, got {:?}", + keys[1] + ); + assert!( + !keys[1].contains('/') && !keys[1].contains(':'), + "URL punctuation must be reduced, got {:?}", + keys[1] + ); + } + + #[test] + fn punctuation_distinct_urls_have_distinct_surrogate_keys() { + let mut slash = key(); + slash.url = "https://example.com/a/b".to_string(); + let mut colon = key(); + colon.url = "https://example.com/a:b".to_string(); + assert_ne!(slash.surrogate_keys()[1], colon.surrogate_keys()[1]); + } + + #[test] + fn an_absent_vary_header_is_distinct_from_an_empty_one() { + // "absent" and "present but empty" are different requests to the origin, so + // they must not share a template. + let spec = VarySpec::new(["RSC".to_string()]); + let absent_headers = http::HeaderMap::new(); + let absent = spec.values_from(&absent_headers); + let mut empty_headers = http::HeaderMap::new(); + empty_headers.insert("rsc", http::HeaderValue::from_static("")); + let empty = spec.values_from(&empty_headers); + assert_ne!(absent, empty); + + // The distinction that does matter: a present value differs from both. + let mut present_headers = http::HeaderMap::new(); + present_headers.insert("rsc", http::HeaderValue::from_static("1")); + let present = spec.values_from(&present_headers); + assert_ne!(present, absent); + } + + #[test] + fn repeated_and_non_utf8_vary_values_are_preserved() { + let spec = VarySpec::new(["x-route".to_string()]); + let mut headers = http::HeaderMap::new(); + headers.append("x-route", http::HeaderValue::from_static("first")); + headers.append( + "x-route", + http::HeaderValue::from_bytes(b"\xffsecond").expect("obs-text is valid field data"), + ); + assert_eq!( + spec.values_from(&headers), + vec![VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"first".to_vec(), b"\xffsecond".to_vec()]), + }] + ); + } + + #[test] + fn vary_spec_lowercases_configured_names() { + assert_eq!( + VarySpec::new(["RSC".to_string(), "Accept-Encoding".to_string()]).names(), + ["rsc"] + ); + } + + #[test] + fn vary_spec_rejects_invalid_names_and_deduplicates_case_insensitively() { + assert_eq!( + VarySpec::try_new(["not a header".to_string()]), + Err("not a header".to_string()) + ); + assert_eq!( + VarySpec::try_new(["RSC".to_string(), "rsc".to_string()]) + .expect("valid names") + .names(), + ["rsc"] + ); + } + + #[test] + fn drift_is_detected_when_the_origin_varies_on_something_unconfigured() { + // The failure mode configured-Vary has: the origin adds a header to its Vary, + // nobody updates config, and requests differing only in that header start + // sharing a template. + let spec = VarySpec::new(["rsc".to_string()]); + + assert!( + spec.uncovered_by(["rsc"]).is_empty(), + "a fully covered Vary is not drift" + ); + assert_eq!( + spec.uncovered_by(["rsc, next-router-prefetch, Accept-Encoding"]), + vec!["next-router-prefetch"], + "uncovered names must be reported so the stale config is identifiable; \ + accept-encoding is excluded because the key covers it structurally" + ); + } + + #[test] + fn a_key_field_counts_as_coverage_without_being_configured() { + // The failure this prevents is silent and total: every compressing origin sends + // `Vary: Accept-Encoding`, so treating it as a gap means the cache never stores + // anything, and a spike measuring hit rate would report ~0 and look like a + // finding rather than a bug. + let spec = VarySpec::new([]); + + assert!( + spec.uncovered_by(["Accept-Encoding"]).is_empty(), + "the shared path uses one upstream encoding offer and stores identity bytes" + ); + assert_eq!( + spec.uncovered_by(["accept-encoding, rsc"]), + vec!["rsc"], + "only the genuinely uncovered name should be reported" + ); + } + + #[test] + fn a_wildcard_vary_is_not_reported_as_a_named_gap() { + // `Vary: *` means uncacheable, which the eligibility gate handles. Reporting + // it here would produce a nonsense "configure a header called *". + let spec = VarySpec::new(["rsc".to_string()]); + assert!(spec.uncovered_by(["*"]).is_empty()); + } + + #[test] + fn metadata_round_trips() { + let metadata = TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: vec![ + ( + "content-security-policy".to_string(), + "default-src 'self'".to_string(), + ), + ( + "content-security-policy".to_string(), + "script-src 'self'".to_string(), + ), + ( + "link".to_string(), + "; rel=preload; as=script".to_string(), + ), + ], + content_type: "text/html; charset=utf-8".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 42, + }; + let decoded = + TemplateMetadata::decode(&metadata.encode()).expect("should decode what it encoded"); + assert_eq!(decoded, metadata); + } + + #[test] + fn unparseable_metadata_is_a_miss_not_a_panic() { + for raw in [ + &b"not-key-value"[..], + &b"v=notanumber\nce=gzip\nct=text/html\nlen=1"[..], + &b"v=1\nce=gzip\nct=text/html"[..], + &b"v=1\nce=gzip\nct=text/html\nlen=1\nunexpected=1"[..], + &b"v=1\nv=1\nce=identity\nct=text/html\nlen=1"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=cache-control:public"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=not-a-policy:value"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=malformed"[..], + &b"v=1\nce=identity\nct=application/json\nlen=1"[..], + &[0xff, 0xfe][..], + ] { + assert_eq!( + TemplateMetadata::decode(raw), + None, + "malformed metadata must be a miss, not a partial read: {raw:?}" + ); + } + } + + #[test] + fn the_policy_allowlist_covers_document_security_and_delivery_headers() { + for required in [ + "strict-transport-security", + "cross-origin-opener-policy", + "cross-origin-embedder-policy", + "cross-origin-resource-policy", + "origin-agent-cluster", + "reporting-endpoints", + "report-to", + "link", + ] { + assert!( + REPLAYABLE_POLICY_HEADERS.contains(&required), + "warm ESI hits must preserve {required}" + ); + } + } + + #[tokio::test] + async fn the_null_object_reports_unsupported_rather_than_failing() { + // Degrading to per-request transformation keeps the shared modes portable on + // adapters with no cache; erroring would make them Fastly-only outright. + let cache = UnavailableTemplateCache; + assert_eq!( + cache.get(&key()).await.err(), + Some(TemplateCacheMiss::Unsupported) + ); + assert!(matches!( + cache + .put( + &key(), + &TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: Vec::new(), + content_type: "text/html".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 0, + }, + Vec::new(), + std::time::Duration::from_secs(1) + ) + .await, + Err(TemplateCacheError::Unsupported) + )); + } +} diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index a39a26430..e9e02e524 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -168,6 +168,16 @@ pub struct RuntimeServices { /// per-request basis by cloning [`RuntimeServices`] with /// [`RuntimeServices::with_kv_store`]. pub(crate) kv_store: Arc, + /// Shared transformed-template cache (C2). Defaults to + /// [`UnavailableTemplateCache`], so adapters without one degrade to transforming + /// per request rather than failing. Spike-only; see + /// [`crate::platform::template_cache`]. + pub(crate) template_cache: Arc, + /// Platform-specific cold-response template assembler. + /// + /// Defaults to [`super::UnavailableTemplateAssembler`]. Core retains a portable + /// byte-seam fallback when this service is unavailable or rejects a document. + pub(crate) template_assembler: Arc, /// Dynamic backend registration and name prediction. pub(crate) backend: Arc, /// Outbound HTTP client abstraction. @@ -223,6 +233,18 @@ impl RuntimeServices { &*self.kv_store } + /// The shared transformed-template cache. Spike-only. + #[must_use] + pub fn template_cache(&self) -> &dyn super::PlatformTemplateCache { + &*self.template_cache + } + + /// Returns the platform-specific cold-response template assembler. + #[must_use] + pub fn template_assembler(&self) -> &dyn super::PlatformTemplateAssembler { + &*self.template_assembler + } + /// Returns the dynamic backend service. #[must_use] pub fn backend(&self) -> &dyn PlatformBackend { @@ -272,6 +294,29 @@ impl RuntimeServices { ..self } } + + /// Returns a clone of this instance with the template cache replaced. + /// + /// Spike-only (#1009). + #[must_use] + pub fn with_template_cache(self, cache: Arc) -> Self { + Self { + template_cache: cache, + ..self + } + } + + /// Returns a clone of this instance with the template assembler replaced. + #[must_use] + pub fn with_template_assembler( + self, + assembler: Arc, + ) -> Self { + Self { + template_assembler: assembler, + ..self + } + } } impl fmt::Debug for RuntimeServices { @@ -290,6 +335,8 @@ pub struct RuntimeServicesBuilder { config_store: Option>, secret_store: Option>, kv_store: Option>, + template_cache: Option>, + template_assembler: Option>, backend: Option>, http_client: Option>, geo: Option>, @@ -303,6 +350,8 @@ impl RuntimeServicesBuilder { config_store: None, secret_store: None, kv_store: None, + template_cache: None, + template_assembler: None, backend: None, http_client: None, geo: None, @@ -325,6 +374,23 @@ impl RuntimeServicesBuilder { self } + /// Set the shared transformed-template cache. Spike-only. + #[must_use] + pub fn template_cache(mut self, cache: Arc) -> Self { + self.template_cache = Some(cache); + self + } + + /// Set the platform-specific cold-response template assembler. + #[must_use] + pub fn template_assembler( + mut self, + assembler: Arc, + ) -> Self { + self.template_assembler = Some(assembler); + self + } + /// Set the KV store implementation. #[must_use] pub fn kv_store(mut self, kv_store: Arc) -> Self { @@ -387,6 +453,14 @@ impl RuntimeServicesBuilder { kv_store: self .kv_store .expect("should set kv_store before building RuntimeServices"), + // Defaulted rather than required: an adapter with no template cache + // should degrade to transforming per request, not fail to build. + template_cache: self + .template_cache + .unwrap_or_else(|| Arc::new(super::UnavailableTemplateCache)), + template_assembler: self + .template_assembler + .unwrap_or_else(|| Arc::new(super::UnavailableTemplateAssembler)), backend: self .backend .expect("should set backend before building RuntimeServices"), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index e2ff3d3cf..65d5dd124 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -21,7 +21,7 @@ use std::borrow::Cow; use std::io::Write; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant, SystemTime}; use brotli::Decompressor; use brotli::enc::BrotliEncoderParams; @@ -51,15 +51,19 @@ use crate::auction::types::{ use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; +use crate::creative_opportunities::{AssemblyMode, CreativeOpportunitiesConfig}; use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; +use crate::html_processor::BodyCloseInjection; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; use crate::integrations::IntegrationRegistry; -use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use crate::platform::{ + GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, VarySpec, +}; use crate::price_bucket::{PriceGranularity, price_bucket}; -use crate::response_privacy::enforce_synthesized_html_cache_privacy; +use crate::response_privacy::{enforce_private_no_store, enforce_synthesized_html_cache_privacy}; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{ @@ -70,6 +74,68 @@ use crate::streaming_replacer::create_url_replacer; const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +const HEADER_X_TS_C2_CACHE: &str = "x-ts-c2-cache"; +const HEADER_X_TS_ASSEMBLY: &str = "x-ts-assembly"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum C2ResponseState { + Hit, + MissReserved, + MissStored, + MissStoreError, + BypassRequest, + BypassResponse, + Unsupported, + Invalid, + BackendError, +} + +impl C2ResponseState { + const fn as_str(self) -> &'static str { + match self { + Self::Hit => "hit", + Self::MissReserved => "miss-reserved", + Self::MissStored => "miss-stored", + Self::MissStoreError => "miss-store-error", + Self::BypassRequest => "bypass-request", + Self::BypassResponse => "bypass-response", + Self::Unsupported => "unsupported", + Self::Invalid => "invalid", + Self::BackendError => "backend-error", + } + } +} + +fn set_c2_response_state(response: &mut Response, state: C2ResponseState) { + response.headers_mut().insert( + HEADER_X_TS_C2_CACHE, + HeaderValue::from_static(state.as_str()), + ); +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum AssemblyResponseState { + EsiParser, + ByteSeamFallback, + ByteSeam, +} + +impl AssemblyResponseState { + const fn as_str(self) -> &'static str { + match self { + Self::EsiParser => "esi-parser", + Self::ByteSeamFallback => "byte-seam-fallback", + Self::ByteSeam => "byte-seam", + } + } +} + +fn set_assembly_response_state(response: &mut Response, state: AssemblyResponseState) { + response.headers_mut().insert( + HEADER_X_TS_ASSEMBLY, + HeaderValue::from_static(state.as_str()), + ); +} fn body_as_reader( body: EdgeBody, @@ -201,11 +267,16 @@ fn restrict_accept_encoding(req: &mut Request) { // origin responds without compression. Adding encodings here would cause the // origin to compress its response even though the client never asked for it, // and the client would then receive content it cannot decode. + if !req.headers().contains_key(header::ACCEPT_ENCODING) { + return; + } let Some(current) = req .headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned) + .get_all(header::ACCEPT_ENCODING) + .iter() + .map(|value| value.to_str().ok()) + .collect::>>() + .map(|values| values.join(", ")) else { return; }; @@ -273,6 +344,158 @@ fn accept_encoding_qvalue(header_value: &str, target: &str) -> Option { matched_qvalue } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReaderEncodingError { + Malformed, + NoAcceptableEncoding, +} + +fn parse_quality_value(value: &str) -> Option { + let value = value.trim(); + let (whole, fraction) = value + .split_once('.') + .map_or((value, None), |(whole, fraction)| (whole, Some(fraction))); + let fraction_is_valid = fraction.is_none_or(|fraction| { + fraction.len() <= 3 && fraction.bytes().all(|byte| byte.is_ascii_digit()) + }); + if !fraction_is_valid { + return None; + } + match whole { + "0" => value.parse().ok(), + "1" if fraction.is_none_or(|fraction| fraction.bytes().all(|byte| byte == b'0')) => { + Some(1.0) + } + _ => None, + } +} + +fn negotiate_reader_compression( + headers: &edgezero_core::http::HeaderMap, +) -> Result { + if !headers.contains_key(header::ACCEPT_ENCODING) { + return Ok(Compression::None); + } + + let mut qualities = Vec::<(String, f32)>::new(); + for field in headers.get_all(header::ACCEPT_ENCODING) { + let field = field.to_str().map_err(|_| ReaderEncodingError::Malformed)?; + for item in field + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + { + let mut parts = item.split(';'); + let token = parts + .next() + .map(str::trim) + .filter(|token| !token.is_empty()) + .ok_or(ReaderEncodingError::Malformed)? + .to_ascii_lowercase(); + if token != "*" && http::HeaderName::from_bytes(token.as_bytes()).is_err() { + return Err(ReaderEncodingError::Malformed); + } + let mut quality = 1.0; + let mut saw_quality = false; + for parameter in parts { + let (name, value) = parameter + .trim() + .split_once('=') + .ok_or(ReaderEncodingError::Malformed)?; + if !name.trim().eq_ignore_ascii_case("q") || saw_quality { + return Err(ReaderEncodingError::Malformed); + } + quality = parse_quality_value(value).ok_or(ReaderEncodingError::Malformed)?; + saw_quality = true; + } + if qualities.iter().any(|(seen, _)| seen == &token) { + return Err(ReaderEncodingError::Malformed); + } + qualities.push((token, quality)); + } + } + + let explicit = |name: &str| { + qualities + .iter() + .find_map(|(candidate, quality)| (candidate == name).then_some(*quality)) + }; + let wildcard = explicit("*"); + let quality_for = |name: &str| explicit(name).or(wildcard).unwrap_or(0.0); + // Identity is implicitly acceptable at q=1 unless explicitly excluded, or a + // wildcard q=0 excludes every unlisted coding. + let identity_quality = + explicit("identity").unwrap_or_else(|| if wildcard == Some(0.0) { 0.0 } else { 1.0 }); + + let candidates = [ + (Compression::Brotli, quality_for("br")), + (Compression::Gzip, quality_for("gzip")), + (Compression::Deflate, quality_for("deflate")), + (Compression::None, identity_quality), + ]; + let mut selected = None; + for (compression, quality) in candidates { + if quality > 0.0 && selected.is_none_or(|(_, best)| quality > best) { + selected = Some((compression, quality)); + } + } + selected + .map(|(compression, _)| compression) + .ok_or(ReaderEncodingError::NoAcceptableEncoding) +} + +fn set_response_compression(response: &mut Response, compression: Compression) { + let encoding = match compression { + Compression::None => None, + Compression::Gzip => Some("gzip"), + Compression::Deflate => Some("deflate"), + Compression::Brotli => Some("br"), + }; + if let Some(encoding) = encoding { + response + .headers_mut() + .insert(header::CONTENT_ENCODING, HeaderValue::from_static(encoding)); + } else { + response.headers_mut().remove(header::CONTENT_ENCODING); + } + let varies_on_encoding = response + .headers() + .get_all(header::VARY) + .iter() + .any(|value| { + value.to_str().is_ok_and(|value| { + value + .split(',') + .any(|name| name.trim().eq_ignore_ascii_case("accept-encoding")) + }) + }); + if !varies_on_encoding { + response + .headers_mut() + .append(header::VARY, HeaderValue::from_static("Accept-Encoding")); + } + response.headers_mut().remove(header::CONTENT_LENGTH); +} + +fn response_compression(response: &Response) -> Compression { + response + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .map(Compression::from_content_encoding) + .unwrap_or(Compression::None) +} + +fn encode_complete_body( + body: Vec, + compression: Compression, +) -> Result, Report> { + let mut encoder = BodyStreamEncoder::new(compression); + let mut encoded = encoder.encode_chunk(body)?; + encoded.extend_from_slice(&encoder.finish()?); + Ok(encoded) +} + /// Unified tsjs static serving: `/static/tsjs=` /// /// Serves two types of bundles: @@ -361,6 +584,8 @@ struct ProcessResponseParams<'a> { suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, + /// See [`HtmlStreamProcessorParams::shared_template_authorized`]. + shared_template_authorized: bool, } struct PublisherBodyProcessor { @@ -384,9 +609,10 @@ impl PublisherBodyProcessor { settings, integration_registry, ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), - ad_bids_state: Arc::clone(¶ms.ad_bids_state), + ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), + shared_template_authorized: params.template_cache_key.is_some(), })?) } else if is_rsc_flight { Box::new(RscFlightUrlRewriter::new( @@ -428,6 +654,7 @@ fn process_response_streaming( body: EdgeBody, output: &mut W, params: &ProcessResponseParams, + output_compression: Compression, ) -> Result<(), Report> { let is_html = is_html_content_type(params.content_type); let is_rsc_flight = @@ -443,7 +670,7 @@ fn process_response_streaming( let compression = Compression::from_content_encoding(params.content_encoding); let config = PipelineConfig { input_compression: compression, - output_compression: compression, + output_compression, chunk_size: 8192, }; // Bound how much decoded gzip output may sit in the heap at once, using the @@ -465,6 +692,7 @@ fn process_response_streaming( ad_bids_state: params.ad_bids_state.clone(), suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.cloned(), + shared_template_authorized: params.shared_template_authorized, })?; StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) @@ -509,13 +737,21 @@ async fn process_response_streaming_async( params.content_encoding ); - let compression = Compression::from_content_encoding(¶ms.content_encoding); + let input_compression = Compression::from_content_encoding(¶ms.content_encoding); + // A C2 template is always identity bytes. Decode during the transform instead of + // recompressing and immediately decoding the entire buffered result afterwards. + let output_compression = if params.template_cache_key.is_some() { + Compression::None + } else { + input_compression + }; let mut processor = PublisherBodyProcessor::new(params, settings, integration_registry)?; process_body_chunks_async( body, output, &mut processor, - compression, + input_compression, + output_compression, settings.publisher.max_buffered_body_bytes, ) .await @@ -557,11 +793,12 @@ async fn process_body_chunks_async( body: EdgeBody, writer: &mut W, processor: &mut P, - compression: Compression, + input_compression: Compression, + output_compression: Compression, max_body_bytes: usize, ) -> Result<(), Report> { - let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); - let mut encoder = BodyStreamEncoder::new(compression); + let mut decoder = BodyStreamDecoder::new(input_compression, max_body_bytes); + let mut encoder = BodyStreamEncoder::new(output_compression); let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_body_bytes); while let Some(segments) = @@ -950,6 +1187,137 @@ struct HtmlStreamProcessorParams<'a> { ad_bids_state: Arc>>, suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option, + /// Whether a shared template was authorized for this response. + /// + /// Carried rather than re-derived so both seams see the same answer. See + /// [`effective_assembly_mode`]. + shared_template_authorized: bool, +} + +/// The diagnostics decision the template may carry. +/// +/// Diagnostics is request-scoped — activated by a cookie or query parameter, and +/// documented as an immutable per-request decision — so it must not reach a shared +/// template. +/// +/// It does not leak today even without this gate, but only by coincidence: +/// `requires_private_no_store()` is a strict superset of the conditions under which +/// a script is emitted, and that stamp lands before the C2 gate reads response +/// headers, so the gate refuses. Two independent conditions that happen to align, +/// with nothing enforcing the relationship. This makes the guarantee explicit; +/// `requires_private_no_store_is_a_superset_of_injection` keeps the coincidence as a +/// backstop if this gate is ever removed. +pub(crate) fn template_gpt_diagnostics( + mode: AssemblyMode, + decision: Option, +) -> Option { + match mode { + AssemblyMode::Inline => decision, + AssemblyMode::Esi => None, + } +} + +/// The marker emitted at the `` seam under [`AssemblyMode::Esi`], reserving the +/// place this reader's slots and bids are spliced into. +/// +/// An inert HTML comment, deliberately. Template schema v1 used an executable ESI include +/// tag here, when the `esi` crate resolved it at the edge. That crate was removed from the +/// render path because it truncates any element larger than its 16 KB chunk size, and +/// nothing has parsed ESI since. What remained was a tag that *looked* executable, would +/// have been executed by any ESI-enabled layer in front of us, and renders as text in a +/// browser if assembly is ever skipped. A comment cannot do any of those things: an +/// unassembled template degrades to a page with no ads rather than a page with a visible +/// tag. +/// +/// Carries no URL. Every byte here is a byte every reader of the shared template +/// receives, so nothing request-scoped may appear, and keeping a URL out also removes +/// any escaping question at the seam. +pub const AD_ASSEMBLY_SEAM: &str = ""; + +/// The mode the operator asked for, before availability is taken into account. +/// +/// Spelled once, because the mode has to mean the same thing at the cache key, at the +/// seam, and at both hit finalizers. Every one of those re-derived it from the same +/// `Option` chain, and the finalizers had no way to ask at all — which is why they +/// demanded a seam marker of a mode that emits none. +fn configured_assembly_mode(settings: &Settings) -> AssemblyMode { + settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::assembly_mode) + .unwrap_or_default() +} + +/// Whether this mode's `` seam emits [`AD_ASSEMBLY_SEAM`]. +/// +/// The property that decides whether a template is *expected* to have a hole in it, and +/// therefore whether the absence of one is a defect or the design. Only `Esi` splices per +/// reader. +/// +/// Matched exhaustively rather than compared against `Esi`, so a new mode has to state +/// its answer here instead of silently inheriting one. +fn mode_emits_seam_marker(mode: AssemblyMode) -> bool { + match mode { + AssemblyMode::Inline => false, + AssemblyMode::Esi => true, + } +} + +/// The assembly mode this response will actually be delivered under. +/// +/// The configured mode says what the operator wants; the cache key says whether it is +/// available. A shared mode with no key means the gate refused this response — the +/// origin set a cookie, declared a `Vary` the key does not cover, returned a non-200, +/// and so on — so there is no shared template to build and nothing downstream will +/// assemble one. +/// +/// When that happens the request falls back to [`AssemblyMode::Inline`] **entirely**, +/// at every seam. Falling back at one seam and not another is what produced the failure +/// this function exists to prevent: the `` seam emitted a legacy ESI tag because +/// the mode was `Esi`, while assembly was skipped because there was no key, so the reader +/// received a document with unresolved executable ESI markup in it and no bids at all. +/// +/// Bypassing is the *normal* case against a real origin, not an edge case, so this path +/// runs far more often than the shared one. +fn effective_assembly_mode(settings: &Settings, shared_template_authorized: bool) -> AssemblyMode { + let configured = configured_assembly_mode(settings); + if matches!(configured, AssemblyMode::Inline) || shared_template_authorized { + return configured; + } + log::debug!( + "assembly mode {configured:?} is unavailable for this response (no shared template \ + was authorized); falling back to inline" + ); + AssemblyMode::Inline +} + +/// What the `` seam should inject, given the assembly mode. +/// +/// Explicit rather than inferred. The previous shape read +/// `ad_slots_script.is_some()` inside the element handler, which silently coupled +/// two independent decisions: once [`template_ad_slots_script`] stopped emitting a +/// head script under a shared mode, body-close injection stopped with it. +/// +/// `Esi` emits [`AD_ASSEMBLY_SEAM`], an inert HTML comment marking where this reader's +/// slots and bids are spliced in. Assembly is a byte split on that comment, performed by +/// this crate on both the miss and the hit path; no ESI layer is involved. +pub(crate) fn body_close_injection( + mode: AssemblyMode, + head_script_present: bool, +) -> BodyCloseInjection { + match mode { + // Per-navigation and never shared, so gating on slot presence is correct. + AssemblyMode::Inline => { + if head_script_present { + BodyCloseInjection::InlineBids + } else { + BodyCloseInjection::None + } + } + // Constant across every request that reaches the transform — which is what + // makes it safe in a shared template. + AssemblyMode::Esi => BodyCloseInjection::Marker(AD_ASSEMBLY_SEAM.to_string()), + } } fn create_html_stream_processor( @@ -963,10 +1331,18 @@ fn create_html_stream_processor( params.origin_host, params.request_host, params.request_scheme, - ) - .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(params.gpt_diagnostics) - .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); + ); + + let assembly_mode = effective_assembly_mode(params.settings, params.shared_template_authorized); + let body_close = body_close_injection(assembly_mode, params.ad_slots_script.is_some()); + + let gpt_diagnostics = template_gpt_diagnostics(assembly_mode, params.gpt_diagnostics); + + let config = config + .with_ad_state(params.ad_slots_script, params.ad_bids_state) + .with_gpt_diagnostics(gpt_diagnostics) + .with_body_close(body_close) + .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); Ok(create_html_processor(config)) } @@ -1000,6 +1376,27 @@ pub enum PublisherResponse { /// Parameters for [`process_response_streaming`]. params: Box, }, + /// A shared template read from C2, to be assembled on the way out. + /// + /// Distinct from [`Self::Stream`] because the bytes are **already transformed** — + /// running them through `lol_html` again would inject a second tsjs `", + html_escape_for_script(slots_json), + html_escape_for_script(&bids) + ) +} + +/// The slot definitions a shared-mode seam must carry, as JSON. +/// +/// Mirrors [`template_ad_slots_script`]'s gating: same `should_run_ad_stack` condition, +/// same slot set. The difference is only *where* it is delivered — the seam, per +/// request, rather than the head, into a shared template. +pub(crate) fn seam_ad_slots_json( + mode: AssemblyMode, + should_run_ad_stack: bool, + settings: &Settings, + matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + request_path: &str, +) -> Option { + if matches!(mode, AssemblyMode::Inline) || !should_run_ad_stack { + return None; + } + let co_config = settings.creative_opportunities.as_ref()?; + let section = co_config.section_for_path(request_path); + let slots: Vec = matched_slots + .iter() + .filter_map(|slot| build_slot_json(slot, co_config, §ion)) + .collect(); + Some( + serde_json::to_string(&slots) + .expect("serde_json::to_string of Vec should be infallible"), + ) +} + /// Build the empty-bids `origin" + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ( + "content-security-policy", + "default-src 'self'; script-src 'nonce-reader-nonce'", + ), + ], + ); + } + + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "a response-bound CSP nonce and its HTML must never be reused from C2" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty() + ); + } + + #[tokio::test] + async fn a_post_is_never_answered_from_a_cached_get() { + // `handle_publisher_request` is the `*`-method fallback route, so a publisher + // path that renders a page on GET and accepts a form or webhook on POST reaches + // here for both. Serving the cached GET to the POST swallows the mutating + // request entirely: the origin never sees it, the caller gets 200 and a page, + // and nothing anywhere reports a problem. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + // Warm the cache with a GET. + let _ = run(&settings, &services, navigation_request()).await; + assert_eq!(stub.recorded_request_uris().len(), 1); + + let post = HttpRequest::builder() + .method(Method::POST) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .body(EdgeBody::from("field=value")) + .expect("should build post request"); + let _ = run(&settings, &services, post).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "the POST must reach the origin rather than being answered from the \ + cached GET" + ); + assert_eq!( + cache.entries.lock().expect("should lock entries").len(), + 1, + "and it must not store a template of its own" + ); + } + + #[tokio::test] + async fn an_authenticated_request_is_not_served_a_shared_template() { + // The stored template is perfectly cacheable; this request is not entitled + // to it. The store gate cannot express that, because it is a property of + // the reader rather than of the bytes — which is why the lookup re-checks. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + assert_eq!( + cache.entries.lock().expect("should lock entries").len(), + 1, + "the cold request should have populated the cache" + ); + + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let authenticated = HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .body(EdgeBody::empty()) + .expect("should build authenticated request"); + let _ = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + authenticated, + ) + .await + .expect("should proxy publisher request"); + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "an authenticated request must reach the origin rather than read a \ + shared template" + ); + } + } + + mod c2_gate_tests { + //! `cache::core` stores whatever it is handed and rejects nothing, so every + //! one of these conditions is the caller's to enforce. Each is a leak vector + //! or an eligibility rule, not a preference. + + use super::*; + use crate::creative_opportunities::AssemblyMode; + use edgezero_core::http::HeaderName; + + fn headers(pairs: &[(HeaderName, &str)]) -> edgezero_core::http::HeaderMap { + let mut map = edgezero_core::http::HeaderMap::new(); + for (name, value) in pairs { + map.insert( + name.clone(), + HeaderValue::from_str(value).expect("should build header value"), + ); + } + map + } + + fn shareable() -> edgezero_core::http::HeaderMap { + headers(&[(header::CACHE_CONTROL, "max-age=60")]) + } + + /// The shipped default: no operator has stated what the origin varies on, so the + /// key covers nothing. Responses without a `Vary` are unaffected; any `Vary` at + /// all disqualifies. + fn nothing_covered() -> VarySpec { + VarySpec::new([]) + } + + #[test] + fn an_unconfigured_deployment_never_caches_a_varying_response() { + // The fail-closed default. An operator who has not stated the origin's Vary + // must not acquire a shared cache by omission — and a real origin varies on + // something, so this is the common path, not an edge case. + // Deliberately not `Accept-Encoding`: the shared path normalizes supported + // content codings to one identity template, so that header is covered + // whatever the operator configured. Using it here would test the + // structural-coverage carve-out rather than the drift guard. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("rsc")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + ¬hing_covered(), + ), + Some(C2BypassReason::VaryNotCovered(VaryGap(vec![ + "rsc".to_string() + ]))), + "an unstated Vary must disqualify rather than silently under-key" + ); + } + + #[test] + fn an_origin_that_varies_on_cookie_is_refused_even_when_declared_independent() { + // The backstop that makes `origin_is_cookie_independent` safe to offer. The + // operator asserts their origin ignores cookies; if the origin then says + // otherwise, the assertion loses. Without this, a wrong assertion would + // silently cross-serve personalized HTML. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("Cookie")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + // The operator's assertion has already been applied here: this is + // `false` precisely because they declared independence. + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["cookie".to_string()]), + ), + Some(C2BypassReason::VaryCookie), + "the origin's declaration must override both cookie independence and an \ + accidentally configured per-cookie key" + ); + } + + #[test] + fn a_private_directive_on_a_second_cache_control_line_is_refused() { + // `HeaderMap::get` returns the first value only. An origin that sends + // `Cache-Control: public, max-age=300` and then `Cache-Control: private` on a + // separate line means exactly what one comma-joined line would mean, but the + // second line was invisible — so a response the origin marked private was + // written to a cache shared between readers. The `Vary` reads a few lines up + // already use `get_all` for the same reason. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("public, max-age=300"), + ); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("private")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "a directive on any Cache-Control line must disqualify the response" + ); + } + + #[test] + fn a_no_store_directive_on_a_second_cache_control_line_is_refused() { + // Same defect, the other directive that matters — `no-store` is the one an + // origin uses for a response that must not be written down anywhere. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable) + ); + } + + #[test] + fn cdn_specific_cache_policy_cannot_be_overridden_by_public_cache_control() { + for name in crate::response_privacy::CDN_CACHE_HEADERS { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("no-store"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "C2 must fail closed on the CDN-specific policy header {name}" + ); + } + } + + #[test] + fn unsupported_vendor_freshness_does_not_authorize_c2() { + for name in crate::response_privacy::CDN_CACHE_HEADERS + .iter() + .filter(|name| **name != "surrogate-control") + { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("max-age=60"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "the Fastly exception must not authorize the vendor policy {name}" + ); + } + } + + #[test] + fn observed_fastly_surrogate_policy_uses_edge_freshness_capped_by_configuration() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200, stale-while-revalidate=21600, stale-if-error=604800", + ), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), + ), + Ok(Duration::from_secs(300)), + "Fastly's edge freshness should take precedence over the shorter browser \ + lifetime, while the configured safety ceiling remains authoritative" + ); + } + + #[test] + fn fastly_surrogate_freshness_takes_precedence_over_standard_freshness() { + for (cache_control, surrogate_control, expected) in [ + ("public, max-age=300", "max-age=30", 30), + ("public, max-age=30", "max-age=300", 300), + ] { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, cache_control), + ( + header::HeaderName::from_static("surrogate-control"), + surrogate_control, + ), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), + ), + Ok(Duration::from_secs(expected)) + ); + } + } + + #[test] + fn surrogate_stale_windows_do_not_extend_fresh_reuse() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=300"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=20, stale-while-revalidate=600, stale-if-error=1200", + ), + (header::AGE, "10"), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), + ), + Ok(Duration::from_secs(10)), + "stale windows are validated metadata, not fresh C2 lifetime" + ); + } + + #[test] + fn ambiguous_or_unsupported_surrogate_policy_fails_closed() { + for (policy, expected) in [ + ("max-age", C2BypassReason::MalformedCachePolicy), + ( + "max-age=30, max-age=60", + C2BypassReason::MalformedCachePolicy, + ), + ("max-age=tomorrow", C2BypassReason::MalformedCachePolicy), + ("max-age=30, public", C2BypassReason::MalformedCachePolicy), + ("stale-if-error=60", C2BypassReason::NoPositiveFreshness), + ("max-age=0", C2BypassReason::NoPositiveFreshness), + ("max-age=30,", C2BypassReason::MalformedCachePolicy), + ] { + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(policy).expect("should build Surrogate-Control"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(expected), + "`{policy}` must fail closed" + ); + } + } + + #[test] + fn restrictive_surrogate_policy_is_never_overridden_by_standard_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(directive).expect("should build Surrogate-Control"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "`{directive}` must remain authoritative" + ); + } + } + + #[test] + fn restrictive_standard_policy_is_never_overridden_by_surrogate_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let publisher_headers = headers(&[ + ( + header::CACHE_CONTROL, + &format!("public, max-age=60, {directive}"), + ), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + ), + ]); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "standard `{directive}` must refuse C2 even with positive edge freshness" + ); + } + } + + #[test] + fn surrogate_control_can_authorize_fastly_edge_freshness_without_browser_freshness() { + let publisher_headers = headers(&[( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + )]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), + ), + Ok(Duration::from_secs(300)), + "Fastly edge freshness should not require browser freshness" + ); + } + + #[test] + fn repeated_cache_control_lines_without_a_disqualifier_still_cache() { + // The other direction: reading every value must not turn an ordinary + // multi-line `Cache-Control` into a bypass, or the fix would disable the + // cache instead of tightening it. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("public")); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + None + ); + } + + #[test] + fn origin_freshness_is_positive_age_adjusted_and_capped() { + let fresh_headers = headers(&[(header::CACHE_CONTROL, "public, max-age=300")]); + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &fresh_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), + ), + Ok(Duration::from_secs(60)) + ); + + let aged = headers(&[ + (header::CACHE_CONTROL, "s-maxage=50, max-age=300"), + (header::AGE, "35"), + ]); + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &aged, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), + ), + Ok(Duration::from_secs(15)) + ); + + let old_date_without_age = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ]); + let one_minute_later = httpdate::parse_http_date("Wed, 12 Aug 2026 08:01:00 GMT") + .expect("should parse fixture time"); + assert_eq!( + origin_shared_ttl_at( + &old_date_without_age, + one_minute_later, + Duration::from_secs(60), + ), + Err(C2BypassReason::NoPositiveFreshness), + "an old Date is apparent age even when an upstream omitted Age" + ); + } + + #[test] + fn zero_exhausted_missing_and_malformed_freshness_are_refused() { + for (map, expected) in [ + ( + headers(&[(header::CACHE_CONTROL, "max-age=0")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=60"), (header::AGE, "60")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "public")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=tomorrow")]), + C2BypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=\"60")]), + C2BypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=+60")]), + C2BypassReason::MalformedCachePolicy, + ), + ] { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(expected) + ); + } + } + + #[test] + fn expires_can_authorize_but_never_extend_an_expired_response() { + let now = httpdate::parse_http_date("Wed, 12 Aug 2026 08:00:00 GMT") + .expect("should parse fixture time"); + let fresh = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); + assert_eq!( + origin_shared_ttl_at(&fresh, now, Duration::from_secs(60)), + Ok(Duration::from_secs(30)) + ); + + let expired = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:01:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); + assert_eq!( + origin_shared_ttl_at(&expired, now, Duration::from_secs(60)), + Err(C2BypassReason::NoPositiveFreshness) + ); + } + + #[test] + fn request_semantics_bypass_c2_except_for_a_max_age_zero_reload() { + for (name, value) in [ + (header::CACHE_CONTROL, "no-cache"), + (header::CACHE_CONTROL, "max-age=30"), + (header::CACHE_CONTROL, "max-age=\"0"), + (header::CACHE_CONTROL, "min-fresh=10"), + (header::CACHE_CONTROL, "no-store"), + (header::PRAGMA, "no-cache"), + (header::PRAGMA, "legacy-extension, no-cache"), + (header::RANGE, "bytes=0-99"), + (header::IF_NONE_MATCH, "\"etag\""), + (header::IF_MODIFIED_SINCE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ] { + let map = headers(&[(name.clone(), value)]); + assert!(request_bypasses_c2(&map), "{name}: {value} must bypass"); + } + assert!( + !request_bypasses_c2(&headers(&[(header::CACHE_CONTROL, "max-age=0")])), + "a browser reload may reuse C2 because the assembled response and auction \ + are still rebuilt for this reader" + ); + assert!(!request_bypasses_c2(&headers(&[( + header::CACHE_CONTROL, + "public" + )]))); + } + + #[test] + fn a_wildcard_vary_is_refused() { + // `VarySpec::uncovered_by` filters `*` out, with a comment saying the + // eligibility gate handles it. It did not — nothing rejected the wildcard, so + // a response the origin said no key can select was shareable. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("*")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + ¬hing_covered(), + ), + Some(C2BypassReason::VaryWildcard) + ); + } + + #[test] + fn a_fully_covered_vary_is_cacheable() { + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, Accept-Encoding"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string(), "accept-encoding".to_string()]), + ), + None, + "a key covering everything the origin varies on is safe to store" + ); + } + + #[test] + fn config_drift_names_the_missing_header() { + // The failure this guards: the origin adds a header to its Vary, nobody + // updates config, and requests differing only in that header start sharing a + // template. The reason must name it, or diagnosing means a bisect. + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, next-router-prefetch"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(C2BypassReason::VaryNotCovered(VaryGap(vec![ + "next-router-prefetch".to_string() + ]))), + "the uncovered header must be named" + ); + } + + #[test] + fn a_vary_split_across_repeated_headers_is_still_checked() { + // Vary is a list header, so an origin may send it once or many times. Reading + // only the first would let the rest through unkeyed. + let mut varying = shareable(); + varying.append(header::VARY, HeaderValue::from_static("rsc")); + varying.append(header::VARY, HeaderValue::from_static("cookie")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(C2BypassReason::VaryCookie), + "a repeated Vary header must not hide names behind the first value" + ); + } + + #[test] + fn a_plain_shareable_html_200_is_cacheable() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + None, + "ESI shareable HTML 200 should be eligible" + ); + } + + #[test] + fn inline_mode_never_writes_a_template() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Inline, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::InlineMode), + "inline has no shared template to write" + ); + } + + #[test] + fn an_authorized_request_is_never_cached() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::AuthorizedRequest), + "an authenticated response must not enter a shared cache" + ); + } + + #[test] + fn a_forwarded_request_cookie_disqualifies_even_without_set_cookie() { + // The dangerous case: session established on an earlier request, so this + // response carries no Set-Cookie, has no Cache-Control at all, is a 200, + // and is HTML — yet is personalized because TS forwarded the Cookie to + // origin unchanged. Every other condition reports it cacheable. + let no_cache_control = edgezero_core::http::HeaderMap::new(); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + true, + StatusCode::OK, + "text/html", + &no_cache_control, + ¬hing_covered(), + ), + Some(C2BypassReason::CookieForwarded), + "cookie-personalized HTML must not become a shared template" + ); + } + + #[test] + fn an_origin_set_cookie_is_never_cached() { + let with_cookie = headers(&[ + (header::CACHE_CONTROL, "max-age=60"), + (header::SET_COOKIE, "sid=abc; Path=/"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &with_cookie, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginSetCookie), + "caching this would replay one visitor's cookie to the next" + ); + } + + #[test] + fn non_shareable_cache_control_is_refused_case_insensitively() { + for directive in [ + "private", + "no-store", + "no-cache", + "Private, max-age=60", + "NO-STORE", + "public, No-Cache", + ] { + let map = headers(&[(header::CACHE_CONTROL, directive)]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "`{directive}` should disqualify the response" + ); + } + } + + #[test] + fn a_datadome_block_is_refused_by_the_status_check() { + // DataDome replaces the document with a 403 + // (`integrations/datadome/protection.rs:778`). There is no separate + // marker to detect, and none is needed. + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::FORBIDDEN, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::NonOkStatus), + "a blocked document must not become the shared template" + ); + } + + #[test] + fn non_html_is_refused() { + for content_type in ["text/x-component", "application/json", ""] { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + content_type, + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::NotHtml), + "`{content_type}` has no HTML template to transform" + ); + } + } + + #[test] + fn unsupported_content_encoding_is_refused_before_representation_headers_change() { + let map = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::CONTENT_ENCODING, "zstd"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::UnsupportedContentEncoding) + ); + + let mut repeated = headers(&[(header::CACHE_CONTROL, "public, max-age=60")]); + repeated.append(header::CONTENT_TYPE, HeaderValue::from_static("text/html")); + repeated.append( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &repeated, + ¬hing_covered(), + ), + Some(C2BypassReason::MalformedRepresentationHeaders) + ); + } + + #[test] + fn leak_vectors_are_reported_before_mere_ineligibility() { + // A response that fails several conditions should name the most serious + // one, so an operator reading the log sees the security reason rather + // than a content-type quibble. + let map = headers(&[ + (header::CACHE_CONTROL, "private"), + (header::SET_COOKIE, "sid=abc"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::FORBIDDEN, + "application/json", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::AuthorizedRequest), + "authorization is the most serious disqualifier and should win" + ); + } + } + + mod template_neutrality_tests { + //! The gate for #1009's shared-template design. + //! + //! An "absence of per-user values" scan is not sufficient here: the bug + //! that nearly shipped was a *conditionally present* element whose own + //! content was per-URL. These tests assert byte-identity across requests + //! that differ only in the gating decision. + + use super::*; + use crate::creative_opportunities::{ + AssemblyMode, CreativeOpportunityFormat, CreativeOpportunitySlot, + }; + + pub(super) fn slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { id: "atf".to_string(), - formats: vec![AdFormat { - media_type: MediaType::Banner, + gam_unit_path: Some("/99999/example/home".to_string()), + div_id: Some("ad-atf".to_string()), + page_patterns: vec!["/**".to_string()], + formats: vec![CreativeOpportunityFormat { width: 300, height: 250, + media_type: MediaType::Banner, }], floor_price: None, targeting: Default::default(), - bidders: Default::default(), - }], - publisher: PublisherInfo { - domain: "test-publisher.com".to_string(), - page_url: Some("https://test-publisher.com/article".to_string()), - }, - user: UserInfo { - id: None, - consent: None, - eids: None, - }, - device: None, - site: None, - context: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + compiled_unit: None, + } } - } - - fn build_request(method: Method, uri: &str) -> HttpRequest { - HttpRequest::builder() - .method(method) - .uri(uri) - .body(EdgeBody::empty()) - .expect("should build test request") - } - - #[test] - fn stream_publisher_body_injects_active_diagnostics_for_materialized_html() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let mut request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/article?ts_console=1") - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build activation request"); - let decision = - crate::integrations::gpt_diagnostics::prepare_request(&settings, &mut request) - .expect("should prepare diagnostics request"); - let mut params = make_stream_params(&settings, ""); - params.content_type = "text/html".to_owned(); - params.gpt_diagnostics = Some(decision); - let mut output = Vec::new(); - - stream_publisher_body( - EdgeBody::from("Example"), - &mut output, - ¶ms, - &settings, - &integration_registry, - ) - .expect("should process materialized HTML"); - - let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); - assert!( - html.contains("__tsjs_gpt_diagnostics_active"), - "should inject the activation flag" - ); - assert!( - html.contains("tsjs-gpt_diagnostics.min.js"), - "should inject the standalone diagnostics module" - ); - } - - #[test] - fn stream_publisher_body_round_trips_gzip() { - let settings = create_test_settings(); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let input = b"{\"asset\":\"https://origin.test-publisher.com/path/file.js\"}"; - let compressed = gzip_encode(input); - let params = make_stream_params(&settings, "gzip"); - let mut output = Vec::new(); - - stream_publisher_body( - EdgeBody::from(compressed), - &mut output, - ¶ms, - &settings, - &integration_registry, - ) - .expect("should stream gzip response through rewrite pipeline"); - let decoded = gzip_decode(&output); - let decoded = String::from_utf8(decoded).expect("should decode rewritten gzip payload"); - assert!( - decoded.contains("https://test-publisher.com/path/file.js"), - "should rewrite origin URLs to the request host" - ); - assert!( - !decoded.contains("origin.test-publisher.com"), - "should remove the origin hostname from the rewritten payload" - ); - } + pub(super) fn settings_with_slots() -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + // Construct the section rather than mutating it if present: the shared + // fixture does not carry `[creative_opportunities]`, and an `if let + // Some(..)` here would silently no-op and make the inline assertion + // below vacuous. + settings.creative_opportunities = Some(CreativeOpportunitiesConfig { + enabled: true, + gam_network_id: "99999".to_string(), + auction_timeout_ms: Some(500), + price_granularity: Default::default(), + section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, + section_segment: None, + slot: vec![slot()], + }); + settings + } - #[test] - fn stream_publisher_body_round_trips_brotli() { - let settings = create_test_settings(); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let input = b"{\"asset\":\"https://origin.test-publisher.com/path/file.css\"}"; - let compressed = brotli_encode(input); - let params = make_stream_params(&settings, "br"); - let mut output = Vec::new(); + #[test] + fn shared_modes_emit_no_head_script_regardless_of_the_gating_decision() { + let settings = settings_with_slots(); + let slots = [slot()]; + let mode = AssemblyMode::Esi; + let ran = template_ad_slots_script(mode, true, &settings, &slots, "/"); + let did_not_run = template_ad_slots_script(mode, false, &settings, &slots, "/"); - stream_publisher_body( - EdgeBody::from(compressed), - &mut output, - ¶ms, - &settings, - &integration_registry, - ) - .expect("should stream brotli response through rewrite pipeline"); + assert_eq!( + ran, did_not_run, + "{mode:?}: the template must be byte-identical whether or not the ad \ + stack ran; a cached object cannot carry one request's consent, bot, \ + prefetch or kill-switch decision" + ); + assert_eq!( + ran, None, + "{mode:?}: adSlots belongs in the per-request seam, not the template" + ); + } - let decoded = brotli_decode(&output); - let decoded = String::from_utf8(decoded).expect("should decode rewritten brotli payload"); - assert!( - decoded.contains("https://test-publisher.com/path/file.css"), - "should rewrite origin URLs to the request host" - ); - assert!( - !decoded.contains("origin.test-publisher.com"), - "should remove the origin hostname from the rewritten payload" - ); - } + #[test] + fn inline_mode_keeps_its_request_dependent_behaviour() { + // Inline responses are per-navigation and never shared, so gating is + // correct there. This guards against "fixing" the shared-mode bug by + // breaking the shipped path. + let settings = settings_with_slots(); + let slots = [slot()]; - #[test] - fn request_ec_uses_cookie_not_header() { - let settings = create_test_settings(); - let header_ec = format!("{}.HdrId1", "a".repeat(64)); - let cookie_ec = format!("{}.CkId01", "b".repeat(64)); - let req = Request::builder() - .method(Method::GET) - .uri("https://test.example.com/page") - .header("x-ts-ec", &header_ec) - .header("cookie", format!("ts-ec={cookie_ec}; other=value")) - .body(EdgeBody::empty()) - .expect("should build test request"); + assert!( + template_ad_slots_script(AssemblyMode::Inline, true, &settings, &slots, "/") + .is_some(), + "inline should emit adSlots when the ad stack runs" + ); + assert_eq!( + template_ad_slots_script(AssemblyMode::Inline, false, &settings, &slots, "/"), + None, + "inline should emit nothing when the ad stack does not run" + ); + } - let ec_context = EcContext::read_from_request(&settings, &req, &noop_services()) - .expect("should read EC context"); + #[test] + fn shared_modes_are_neutral_across_differing_slot_matches() { + // Slot matching folds in the request path. Under a shared mode even + // that must not reach the template. + let settings = settings_with_slots(); - assert_eq!( - ec_context.ec_value(), - Some(cookie_ec.as_str()), - "should resolve request EC ID from cookie" - ); - assert!( - ec_context.cookie_was_present(), - "should detect cookie was present" - ); - assert_eq!( - ec_context.existing_cookie_ec_id(), - Some(cookie_ec.as_str()), - "should return cookie EC value for revocation" - ); - } + let matched = template_ad_slots_script( + AssemblyMode::Esi, + true, + &settings, + &[slot()], + "/news/article", + ); + let unmatched = + template_ad_slots_script(AssemblyMode::Esi, true, &settings, &[], "/other"); - /// Drive `handle_publisher_request` with no creative opportunities — a plain - /// proxy with no server-side auction. Hides the auction/EC wiring so callers - /// read like a simple `(settings, services, req)` proxy. - async fn run_publisher_proxy( - settings: &Settings, - services: &RuntimeServices, - req: Request, - ) -> PublisherResponse { - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let mut ec_context = - EcContext::read_from_request(settings, &req, services).expect("should read EC context"); - handle_publisher_request( - settings, - services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[], - registry: None, - }, - req, - ) - .await - .expect("should proxy publisher request") + assert_eq!( + matched, unmatched, + "the template must not vary with slot matching under a shared mode" + ); + } } mod ssat_cache_policy_tests { @@ -5024,6 +11633,7 @@ mod tests { match response { PublisherResponse::Buffered(response) | PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } | PublisherResponse::PassThrough { response, .. } => response.into_parts().0, } } @@ -5353,7 +11963,9 @@ mod tests { // Assert let response = match response { PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { panic!("unexpected origin 304 should return a buffered response") } }; @@ -5436,7 +12048,9 @@ mod tests { // Assert let response = match response { PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { panic!("noneligible origin 304 should remain buffered") } }; @@ -5510,7 +12124,8 @@ mod tests { *response.body_mut() = body; response } - PublisherResponse::Stream { response, .. } => response, + PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } => response, }; assert_eq!(response.status(), StatusCode::OK); @@ -6222,7 +12837,7 @@ mod tests { read_count: Arc::clone(&read_count), body_close_processed_at: Arc::clone(&body_close_processed_at), }; - let ad_bids_state = Arc::new(Mutex::new(None)); + let ad_bids_state = AdBidsState::default(); let ctx = AuctionCollectCtx { dispatched, telemetry: AuctionTelemetryCarry { @@ -6267,7 +12882,7 @@ mod tests { let settings = create_test_settings(); let services = noop_services(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let ad_bids_state = Arc::new(Mutex::new(None)); + let ad_bids_state = AdBidsState::default(); let mut state = AuctionHoldState::new( DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( test_auction_request(), @@ -6316,6 +12931,7 @@ mod tests { ); assert!( ad_bids_state + .script_cell() .lock() .expect("should lock bid state") .is_none(), @@ -6333,6 +12949,7 @@ mod tests { ); assert!( ad_bids_state + .script_cell() .lock() .expect("should lock bid state") .is_some(), @@ -6764,6 +13381,59 @@ mod tests { ); } + #[test] + fn esi_reader_encoding_negotiation_honours_quality_identity_and_repeated_fields() { + let headers = |values: &[&str]| { + let mut headers = edgezero_core::http::HeaderMap::new(); + for value in values { + headers.append( + header::ACCEPT_ENCODING, + HeaderValue::from_str(value).expect("should build accept-encoding"), + ); + } + headers + }; + + assert_eq!( + negotiate_reader_compression(&headers(&[])), + Ok(Compression::None) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.8", "br;q=0.4, identity;q=0.1"])), + Ok(Compression::Gzip) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip, br"])), + Ok(Compression::Brotli), + "server preference breaks an equal-quality tie" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.5"])), + Ok(Compression::None), + "implicit identity has q=1" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["zstd, identity;q=0"])), + Err(ReaderEncodingError::NoAcceptableEncoding) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=invalid"])), + Err(ReaderEncodingError::Malformed) + ); + for malformed in [ + "gzip;q=1e-1", + "gzip;q=0.1234", + "gzip;q=1.001", + "not a coding;q=1", + ] { + assert_eq!( + negotiate_reader_compression(&headers(&[malformed])), + Err(ReaderEncodingError::Malformed), + "{malformed} is not valid Accept-Encoding syntax" + ); + } + } + #[test] fn tsjs_dynamic_returns_not_found_for_unknown_filename() { let settings = create_test_settings(); @@ -6982,6 +13652,9 @@ mod tests { let body = EdgeBody::from(compressed); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -6989,7 +13662,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7031,6 +13704,9 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7038,7 +13714,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7069,6 +13745,9 @@ mod tests { let registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7076,7 +13755,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7185,6 +13864,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7192,7 +13874,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7239,6 +13921,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7246,7 +13931,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7296,6 +13981,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "deflate".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7303,7 +13991,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7353,6 +14041,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "br".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7360,7 +14051,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7410,6 +14101,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "br".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7417,7 +14111,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7455,6 +14149,9 @@ mod tests { fn non_html_stream_params(content_encoding: &str) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7462,7 +14159,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7642,8 +14339,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let state = Arc::new(Mutex::new(None)); + let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7707,8 +14407,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let state = Arc::new(Mutex::new(None)); + let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7776,6 +14479,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7783,7 +14489,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: Some(test_auction_request()), dispatched_auction: Some(DispatchedAuction::empty_for_test( @@ -7836,6 +14542,9 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7843,7 +14552,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7970,6 +14679,9 @@ mod tests { dispatched_auction: Option, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7980,7 +14692,7 @@ mod tests { r#""# .to_string(), ), - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: dispatched_auction.as_ref().map(|_| test_auction_request()), dispatched_auction, @@ -8316,6 +15028,9 @@ mod tests { let ec_context = EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8323,7 +15038,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: Some(AuctionObservationContext::from_parts( AuctionSource::SpaNavigation, "proxy.example.com", @@ -8499,6 +15214,9 @@ mod tests { .map(bytes::Bytes::copy_from_slice) .collect(); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8509,7 +15227,7 @@ mod tests { r#""# .to_string(), ), - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: Some(test_auction_request()), dispatched_auction: Some(DispatchedAuction::empty_for_test( @@ -8568,8 +15286,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let bids_script = r#""#; - let state = Arc::new(Mutex::new(Some(bids_script.to_string()))); + let state = AdBidsState::with_script(bids_script); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8624,6 +15345,9 @@ mod tests { // Claim gzip encoding but feed non-gzip bytes. The GzDecoder will // error as soon as it tries to read the gzip header. let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8631,7 +15355,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8733,6 +15457,9 @@ mod tests { let body = EdgeBody::from(html.to_vec()); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8740,7 +15467,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8791,6 +15518,9 @@ mod tests { // Small, single-fragment RSC script — placeholder path (not fallback). let html = br#""#; let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8798,7 +15528,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8835,8 +15565,9 @@ mod tests { #[cfg(test)] mod creative_opportunities_tests { use super::super::{ - MatchedSlotsContext, build_ad_slots_script, build_auction_request, build_bid_map, - build_bids_script, diagnostics_auction_id, html_escape_for_script, write_bids_to_state, + AdBidsState, MatchedSlotsContext, build_ad_slots_script, build_auction_request, + build_bid_map, build_bids_script, diagnostics_auction_id, html_escape_for_script, + write_bids_to_state, }; use crate::auction::types::{ApsRendererV1, ApsTagType, Bid, BidRenderer, MediaType}; use crate::consent::ConsentContext; @@ -8862,6 +15593,10 @@ mod tests { auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, section_segment: None, slot: Vec::new(), } @@ -9147,7 +15882,7 @@ mod tests { ), ); - let state = std::sync::Arc::new(std::sync::Mutex::new(None)); + let state = AdBidsState::default(); write_bids_to_state( &winning_bids, PriceGranularity::Dense, @@ -9158,6 +15893,7 @@ mod tests { Some(&auction_request.id), ); let script = state + .script_cell() .lock() .expect("should lock initial bid state") .clone() @@ -9192,6 +15928,7 @@ mod tests { Some(&auction_request.id), ); let empty_script = state + .script_cell() .lock() .expect("should lock empty initial bid state") .clone() diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 21ba9f20b..0fe7650dd 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -9,7 +9,7 @@ //! cache such as Cloudflare would otherwise serve an operator/origin //! `Cache-Control: public` on a cookie-bearing response as-is. -use edgezero_core::http::{HeaderName, HeaderValue, Response, header}; +use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, Response, header}; use crate::settings::Settings; @@ -30,21 +30,67 @@ fn strip_cdn_cache_headers(response: &mut Response) { } } -/// Forces synthesized HTML to be private and non-storable. +/// Whether `Cache-Control` already forbids shared caching. /// -/// Use this exact policy whenever Trusted Server changes an origin HTML -/// representation with request-specific content: force `private, no-store`, -/// remove origin validators, and remove all CDN-targeted cache directives. -pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { +/// Extracted because both arms of the cookie-privacy net below need it. +/// +/// `publisher::c2_bypass_reason` deliberately does **not** call this and keeps its own +/// copy: it additionally treats `no-cache` as non-shareable, because "revalidate before +/// reuse" is correct for an HTTP cache and too permissive for a spike-owned one. The +/// duplicate is the stricter of the two, so consolidating them would loosen the shared- +/// template gate rather than tidy it. +/// +/// Directives are case-insensitive (RFC 9111 §5.2), so `No-Store` and `Private` +/// count. `no-cache` deliberately does **not**: it requires revalidation before +/// reuse, not a refusal to store, so a `no-cache` response is still shareable. +/// Callers needing the stricter reading must check it themselves. +#[must_use] +pub fn is_private_or_no_store(headers: &HeaderMap) -> bool { + headers.get_all(header::CACHE_CONTROL).iter().any(|value| { + value.to_str().is_ok_and(|value| { + value.split(',').any(|directive| { + let name = directive + .split_once('=') + .map_or(directive, |(name, _)| name); + matches!( + name.trim().to_ascii_lowercase().as_str(), + "private" | "no-store" + ) + }) + }) + }) +} + +/// Reassert the terminal privacy invariant for a synthesized per-reader response. +/// +/// Call this after every configurable response mutation. It deliberately overwrites +/// `Cache-Control` and strips validators, expiry metadata, and CDN-specific cache +/// directives so a later integration cannot turn an assembled document into C3. +pub fn enforce_private_no_store(response: &mut Response) { response.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static("private, no-store"), ); - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); + for name in [ + header::ETAG.as_str(), + header::LAST_MODIFIED.as_str(), + header::EXPIRES.as_str(), + header::AGE.as_str(), + ] { + response.headers_mut().remove(name); + } strip_cdn_cache_headers(response); } +/// Forces synthesized HTML to be private and non-storable. +/// +/// Use this exact policy whenever Trusted Server changes an origin HTML +/// representation with request-specific content: force `private, no-store`, +/// remove origin validators, and remove all CDN-targeted cache directives. +pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { + enforce_private_no_store(response); +} + /// Forces cookie-bearing responses to stay private to shared caches. /// /// Any response that sets a per-user cookie (notably the EC identity cookie) @@ -63,14 +109,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. strip_cdn_cache_headers(response); - // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match - // against a lowercased copy — `No-Store` / `Private` must count. - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let already_uncacheable = is_private_or_no_store(response.headers()); if !already_uncacheable { response.headers_mut().insert( header::CACHE_CONTROL, @@ -96,12 +135,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: &mut Response) { enforce_set_cookie_cache_privacy(response); - let response_is_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let response_is_uncacheable = is_private_or_no_store(response.headers()); for (key, value) in &settings.response_headers { if response_is_uncacheable @@ -296,6 +330,40 @@ mod tests { } } + #[test] + fn terminal_private_stamp_removes_every_cache_and_validator_header() { + let mut response = response_builder() + .header(header::CACHE_CONTROL, "public, s-maxage=600") + .header(header::ETAG, "\"origin\"") + .header(header::LAST_MODIFIED, "Wed, 12 Aug 2026 00:00:00 GMT") + .header(header::EXPIRES, "Wed, 12 Aug 2026 01:00:00 GMT") + .header(header::AGE, "30") + .header("surrogate-control", "max-age=600") + .header("cdn-cache-control", "public, max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + enforce_private_no_store(&mut response); + + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store" + ); + for name in [ + header::ETAG.as_str(), + header::LAST_MODIFIED.as_str(), + header::EXPIRES.as_str(), + header::AGE.as_str(), + "surrogate-control", + "cdn-cache-control", + ] { + assert!( + !response.headers().contains_key(name), + "terminal private stamp must strip {name}" + ); + } + } + #[test] fn applies_operator_headers_on_cookieless_response() { let settings = settings_with_response_headers(&[("x-operator", "value")]); diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 0c68d43fe..624b4ef92 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -452,8 +452,18 @@ export interface TsjsApi { * Lives in the bundle so the lifecycle is executable under test and shares * [`navGeneration`] with the SPA auction hook; `gpt_bootstrap.js` installs * a minimal fallback for pages where the bundle fails to load. + * + * `initialSlots` exists for the shared-template `` seam, which is the + * only place slot definitions arrive with the bids rather than from the head + * script. Passing them here rather than assigning `tsjs.adSlots` before the + * call puts them behind the same generation guard: an assignment made ahead + * of the guard would clobber a committed SPA navigation's slots with the SSR + * document's, and then be read by that route's `adInit()`. */ - scheduleInitialAdInit?: (initialBids?: Record) => void; + scheduleInitialAdInit?: ( + initialBids?: Record, + initialSlots?: AuctionSlot[] + ) => void; /** Read-only GPT lifecycle diagnostics API, present only in an activated tab. */ gptDiagnostics?: GptDiagnosticsApi; /** diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index ebf7e225e..bfafd6e6a 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -665,7 +665,15 @@ function installInitialLoadDetector(ts: TsjsApi): void { * SSR bootstrap as current. For the same reason the initial bids payload is * passed in and applied here, generation-guarded — assigning it * unconditionally at body end would clobber the live bids a faster SPA - * navigation already applied. When a navigation has committed since — or + * navigation already applied. + * + * `initialSlots` is passed in for exactly the same reason and was missing it. + * Only the shared-template `` seam sends slots — under `inline` they + * come from the head script, which runs before any navigation can commit — and + * that seam assigned `tsjs.adSlots` on the line *before* calling this. The + * guard protected the bids and `adInit()` while the assignment it was meant to + * protect had already happened, so a committed SPA navigation kept its bids and + * lost its slots. When a navigation has committed since — or * commits while the deferred callback is pending — the SSR payload is * dropped and `adInit()` is not run: running anyway would re-run the newer * route's live slots/bids, destroying and redefining that route's TS slots @@ -685,8 +693,12 @@ function installInitialLoadDetector(ts: TsjsApi): void { * holds whenever the request is actually issued. */ function installScheduleInitialAdInit(ts: TsjsApi): void { - ts.scheduleInitialAdInit = function (initialBids?: Record) { + ts.scheduleInitialAdInit = function ( + initialBids?: Record, + initialSlots?: AuctionSlot[] + ) { if ((ts.navGeneration ?? 0) !== 0) return; + if (initialSlots) ts.adSlots = initialSlots; if (initialBids) ts.bids = initialBids; const runUnlessNavigated = (): void => { if ((ts.navGeneration ?? 0) !== 0) return; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index d3e1d7099..69cb65bfc 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -159,6 +159,36 @@ describe('gpt_bootstrap.js fallback', () => { expect(adInit).not.toHaveBeenCalled(); }); + it('fallback scheduler guards the SSR slot definitions with the same generation check', () => { + // The shared-template seam hands slots to the scheduler rather than assigning + // them itself, so the fallback has to honour the same guard as the bundle. If it + // applied them unconditionally, a page whose bundle failed to load would take the + // stale SSR slots over a committed navigation's. + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + const liveSlot = { + id: 'live_slot', + gam_unit_path: '/123/live', + div_id: 'div-live', + formats: [[300, 250]] as Array<[number, number]>, + }; + const ssrSlot = { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]] as Array<[number, number]>, + }; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + expect(ts.adSlots).toEqual([ssrSlot]); + + ts.adSlots = [liveSlot]; + ts.navGeneration = 1; + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + expect(ts.adSlots).toEqual([liveSlot]); + }); + it('fallback adInit defines, targets, and displays a TS slot through the command queue', () => { const mockSlot = { addService: vi.fn().mockReturnThis(), diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts index 999c60c55..830bac217 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts @@ -310,6 +310,69 @@ describe('scheduleInitialAdInit', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('applies the SSR slot definitions on the initial document', async () => { + // Under a shared-template mode the head script emits no `tsjs.adSlots`, so the + // `` seam is the only source of slot definitions. They must arrive, or + // `adInit()` iterates an empty list and the page defines no TS slots at all. + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + const ssrSlot = { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]] as Array<[number, number]>, + }; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + + expect(ts.adSlots).toEqual([ssrSlot]); + expect(ts.bids).toEqual({ ssr_slot: { hb_pb: '1.00' } }); + }); + + it('drops the SSR slot definitions when a navigation has already committed', async () => { + // The guard covered the bids and the adInit call, but the shared-template seam + // assigned `tsjs.adSlots` on the line *before* calling the scheduler — outside the + // guard entirely. A navigation that committed while the SSR document was still + // streaming therefore kept its own bids and silently lost its slots to the stale + // SSR payload, and the next `adInit()` for that route defined the wrong slots. + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/b'); + await flushAsync(); + expect(ts.navGeneration).toBe(1); + const liveSlot = { + id: 'live_slot', + gam_unit_path: '/123/live', + div_id: 'div-live', + formats: [[300, 250]] as Array<[number, number]>, + }; + ts.adSlots = [liveSlot]; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ + { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]], + }, + ]); + + expect(ts.adSlots).toEqual([liveSlot]); + + window.dispatchEvent(new Event('load')); + flushFrame(); + flushFrame(); + expect(adInit).not.toHaveBeenCalled(); + }); + it('cancels queued GPT work when a navigation commits before the command queue drains', async () => { // adInit() only queues its slot work on googletag.cmd, which drains when // GPT itself loads — possibly long after the generation check that diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index b763cac24..0b94587c4 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1382,6 +1382,122 @@ loader: TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false ``` +### Shared template assembly (`assembly_mode = "esi"`) + +`assembly_mode` controls how initial-page slot and bid state is delivered: + +- `inline` (default) transforms every origin response and injects the current + reader's slots and bids directly. +- `esi` opts into a reader-neutral transformed-template cache on Fastly. The + cache stores identity bytes containing one inert, versioned comment. On an + authorized cold miss, Fastly replaces that comment in a private working copy + with one synthetic ESI include and resolves it from the already-built reader + state using the pinned `stackpop/esi` parser. No HTTP fragment request occurs. + Warm hits use an exact byte split instead, preserving the fast article-prefix + stream while the auction finishes. + +This is deliberately not general publisher-controlled ESI. A transformed origin +document containing any ` **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Turn off the redundant origin cache bypass that the spec identifies as the +actual TTFB cost, behind an operator flag, and establish the measurement baseline that +later work is compared against. + +> **This plan does not close #1009.** It contains no ESI arm and no client-fill arm, so +> completing it cannot answer whether ESI separates cacheable content from per-user +> state. It is a **supporting optimisation and the experimental control** for +> [the ESI validation spike](./2026-08-10-1009-esi-validation-spike.md), which is where +> #1009 is actually decided. Scoped and framed this way after external review on +> 2026-08-10. + +**Architecture:** Two investigation tasks that produce recorded findings and no code; one +code task that adds a config-gated timing log and makes the cache bypass operator- +controlled; and one config change that flips it, gated on the first investigation. +Nothing here touches the auction, the `` hold, or bid delivery — those are +Stages 1–2 in the spec and are explicitly out of scope. + +**Tech Stack:** Rust 2024 edition, `wasm32-wasip1`, Fastly Compute, `web_time::Instant` +for wasm-safe timing, `log` for instrumentation, Viceroy for adapter tests. + +**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +(§3 Steps A/B/C and §4 Stage 0). Read §4 and §5 before starting Task 5. + +**Before pushing, run both documentation gates:** + +```bash +cd docs && npm run format && npm run build && cd .. +``` + +`npm run build` is not optional — `format` passes on documents with dead links, and that +shipped a broken docs build on this branch once already. + +**Two prettier gotchas, both hit while writing this plan.** CI gate 7 +(`cd docs && npm run format`) runs `prettier --check` over all of `docs/`, so both bite. + +1. **Not idempotent on embedded markdown fences.** The first `--write` reformats the + outer document and the embedded ` ```markdown ` block only settles on a second pass. + If `--check` still warns immediately after a `--write`, run `--write` again before + concluding anything is wrong. +2. **It mangles bare `snake_case` identifiers inside fences**, reading the underscores as + emphasis and rewriting `origin_fetch_ms` to `origin*fetch_ms`. **Always wrap + identifiers in backticks**, including inside fenced blocks and table cells. + +--- + +## Background an implementer needs + +Trusted Server proxies a publisher's origin, rewrites the HTML at the edge to inject ad +slot definitions and a JS bundle, and runs a server-side ad auction. For requests that +are eligible for that ad stack, `publisher.rs` currently does three things to the origin +request and response that together make the page uncacheable: + +1. strips conditional and range headers so the origin must return a full body, +2. sets a **cache bypass** so the Fastly read-through cache is skipped entirely, and +3. strips every cacheability header from the response. + +The spec establishes that (2) is redundant given (1) — by the time the request reaches +the cache it is already unconditional, so a cache HIT returns a full body anyway — and +that (2) is the dominant cost. This plan makes (2) operator-controlled and then turns it +off, after first confirming that is safe. + +**Why it might not be safe:** RSC (React Server Component) requests and ordinary HTML +navigations share the same URL and are distinguished only by request headers. RSC +requests are not classified as navigations, so they already flow through the cache while +HTML navigations bypass it. Removing the bypass puts both under one cache key. If the +origin does not declare `Vary` for those headers, the cache could serve one +representation in response to a request for the other. Task 1 checks this. + +**Terms:** _POP_ = Fastly edge point of presence. _shield_ = a designated POP that +backs other POPs. _read-through cache_ = Fastly's cache on the backend request path. +_bypass / `Pass`_ = skip that cache. + +--- + +## File structure + +| File | Responsibility in this plan | +| ---------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` | **Create.** Recorded output of Tasks 1–2. Gates Task 5. | +| `crates/trusted-server-core/src/publisher.rs` | **Modify.** Timing log and the bypass flag (Task 3); tests (Task 5). | +| `crates/trusted-server-core/src/settings.rs` | **Modify.** `publisher.bypass_origin_cache` and `debug.publisher_timing` (Task 3). | +| `trusted-server.example.toml` | **Modify.** Document the new key (Task 5). | + +No new modules. No adapter changes: the `bypass_cache` platform capability and its +per-adapter mappings stay in place and keep their tests — the publisher-path call site +becomes operator-controlled rather than unconditional. + +## Task order and dependencies + +Only one edge is real. Do not serialize the rest. + +``` +Task 1 (origin Vary check) ──────┬──> Task 2 (appends to the findings file Task 1 creates) + │ + ├──> Task 5 (flip the flag) +Task 3 (instrumentation + flag) ─┘ +``` + +**Task 1 is externally blocked.** It needs the publisher origin hostname, which lives in +the operator's gitignored `trusted-server.toml`. Arrange access before starting, or the +plan stalls on its first step. + +Task 3 is independent and can start immediately. Task 2 only needs Task 1 far enough to +have created the findings document. Task 5 needs Task 1's verdict **and** Task 3's config +flag to exist. + +--- + +## Task 1: Step A — origin `Vary` check + +**Files:** + +- Create: `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` + +This task is an investigation. It writes no code and gates Task 5. + +- [ ] **Step 1: Get the origin URL** + +The publisher origin is operator config, not in the repo. Read it from the deployed +service config or ask the operator. Do **not** hardcode it into any committed file — the +findings document records the _result_, not the hostname. + +```bash +# The key is `publisher.origin_url` in the operator's trusted-server.toml +# (gitignored). Confirm the value before proceeding. +``` + +- [ ] **Step 2: Request the HTML representation and capture `Vary`** + +```bash +curl -sS -D - -o /dev/null "https:///" \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Accept: text/html' +``` + +Expected: response headers. Record whether a `Vary` header is present and its value. + +- [ ] **Step 3: Request the RSC representation at the same URL** + +```bash +curl -sS -D - -o /dev/null "https:///" \ + -H 'RSC: 1' \ + -H 'Accept: text/x-component' +``` + +Expected: a different `Content-Type` (`text/x-component`) than Step 2, proving the two +representations share a URL. Record `Vary` again. + +- [ ] **Step 4: Probe the `Next-Router-*` headers** + +Do not skip this. The PASS criterion below names these headers, and an implementer who +tests only HTML and `RSC` can record a PASS that is wrong — which routes to Task 5a, the +one outcome this plan calls dangerous. + +```bash +for H in 'Next-Router-Prefetch: 1' 'Next-Router-State-Tree: %5B%22%22%5D'; do + echo "--- $H" + curl -sS -D - -o /dev/null "https:///" \ + -H 'RSC: 1' -H "$H" \ + | grep -iE '^(vary|content-type|content-length|cache-control|set-cookie):' +done +``` + +Compare `Content-Type` and `Content-Length` against the plain `RSC: 1` request from +Step 3. If either differs, the origin varies on that header and `Vary` must name it. + +Capture `Cache-Control` and `Set-Cookie` on every request in this task, not just this +one — see Step 5. + +- [ ] **Step 5: Probe cookie personalization — the bigger hole** + +The representation check above covers RSC-vs-HTML. It does **not** cover the larger +class: TS forwards client cookies to origin unchanged, so any cookie-personalized HTML +(logged-in state, paywall meter, publisher-side A/B assignment) becomes cross-servable +once the cache is on. + +**Do not compare body hashes.** Verified on the live origin: this page regenerates +~170 ad-slot container IDs as fresh 32-hex UUIDs on every request, so three requests give +three different hashes with byte-identical lengths, cookie or not. A hash comparison +reports a false FAIL every time. + +Normalize per-request identifiers, establish the no-cookie baseline drift first, then ask +whether the cookie arm differs by _more_ than that baseline: + +```bash +ORIGIN="https://"; HOSTH="Host: " +norm() { sed -E 's/[0-9a-f]{32}/UUID/g' "$1"; } + +for n in a b; do + curl -sS "$ORIGIN/" -H "$HOSTH" \ + -H 'Sec-Fetch-Dest: document' -H 'Accept: text/html' > "nc_$n.html" +done +curl -sS "$ORIGIN/" -H "$HOSTH" \ + -H 'Sec-Fetch-Dest: document' -H 'Accept: text/html' \ + -H 'Cookie: ' > ck.html + +echo "baseline drift: $(diff <(norm nc_a.html) <(norm nc_b.html) | grep -c '^[<>]')" +echo "with cookie: $(diff <(norm nc_a.html) <(norm ck.html) | grep -c '^[<>]')" +diff <(norm nc_a.html) <(norm ck.html) | head -20 +``` + +Send the `Host` override — the origin is a shared vhost and will not return the right +document without it. Read it from `publisher.origin_host_header_override`. + +**Step A has been run once and returned a PROVISIONAL PASS**, which is **not** sufficient +to flip the flag. See [the findings](./2026-08-08-1009-measurement-findings.md) for the +five untested conditions. Complete them and record a `FINAL PASS` before Task 5. + +Three failure shapes, any of which blocks Stage 0 independently of the `Vary` verdict: + +- Bodies differ by cookie **and** `Vary` does not name `Cookie` → cross-serving of + personalized HTML. +- Origin emits `Set-Cookie` alongside a shared-cacheable `Cache-Control` → the cache can + replay one visitor's cookie to the next. TS's privacy net does not help; it downgrades + **TS's** response, after the cache has already stored the origin's. +- The deployment is `Authorization`-gated (as #1009 describes) and authorized responses + are cacheable → same problem, different header. + +- [ ] **Step 6: Request with the experiment header, if the operator uses one** + +Repeat Step 2 with the publisher's experiment header set to two different values. +Record whether the bodies differ and whether `Vary` names that header. + +- [ ] **Step 7: Record the finding** + +Create `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md`: + +```markdown +# #1009 measurement findings + +## Step A — origin `Vary` declaration + +**Date:** · **Checked by:** + +| Representation | `Content-Type` returned | `Content-Length` | `Vary` present? | `Vary` value | +| --------------------- | ----------------------- | ---------------- | --------------- | ------------ | +| HTML navigation | | | | | +| RSC | | | | | +| RSC + `Next-Router-*` | | | | | +| Experiment variant | | | | | + +**Cookie / auth exposure:** bodies differ by cookie? `Vary: Cookie` present? origin +`Set-Cookie` on a shared-cacheable response? `Authorization`-gated responses cacheable? + +**Verdict:** FINAL PASS / PROVISIONAL PASS / FAIL + +`FINAL PASS` = `Vary` names every request header the origin varies on (`RSC`, any +`Next-Router-*` or experiment header whose value changed the body, **and `Cookie` if +bodies differ by cookie**), no `Set-Cookie` rides a shared-cacheable response, **and** all +five conditions in Task 5's gate are recorded — a real authenticated session cookie, Basic +Auth through TS, the experiment variant, representative routes, and cached-hit +slot/render attribution. + +`PROVISIONAL PASS` = the `Vary` and cookie checks hold, but one or more of those five is +untested. **Not a release gate.** A first pass lands here. + +`FAIL` = any `Vary` or `Set-Cookie` criterion is unmet. + +**Consequence:** `FINAL PASS` → Task 5a (flip the flag). `PROVISIONAL PASS` → close the +gaps before Task 5 starts. `FAIL` → Task 5b (cache-key discriminator). See spec §4. + +**A FAIL is also a live production defect, not only a Stage 0 blocker.** RSC fetches are +not navigations, so they never set the bypass and **already transit the read-through +cache today**. If the origin varies undeclared on `Next-Router-*`, TS is cross-serving RSC +variants in production right now. File it immediately rather than deferring with Task 5b. +``` + +- [ ] **Step 8: Commit** + +CI gate 7 runs `prettier --check` across all of `docs/`, so format the findings file +before staging it — a filled-in markdown table will not be prettier-clean by hand. + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record origin Vary findings for #1009 Stage 0 gate" +``` + +--- + +## Task 2: Step B — what consumes TS's own response headers + +**Files:** + +- Modify: `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` — **created by + Task 1 Step 6.** If Task 1 has not reached that step, create the file with just its + `# #1009 measurement findings` heading rather than blocking. + +Investigation. Determines whether the spec's Stage 3b has a consumer. Does not gate +Task 5, but it appends to Task 1's findings document — do not run the two concurrently +against that file. + +- [ ] **Step 1: Pick a path that already emits shared-cache headers** + +`serve_static_with_etag` emits `public, max-age=300, s-maxage=300` plus +`Surrogate-Control` — see `crates/trusted-server-core/src/http_util.rs:294-311`. It backs +the `/static/tsjs=` bundle route (`publisher.rs:303`, `:322`). Use that URL against +the deployed service. + +- [ ] **Step 2: Request it twice and inspect for cache markers** + +```bash +URL="https:///static/tsjs=" +curl -sS -D - -o /dev/null "$URL" | grep -iE 'x-cache|age:|x-served-by|hit-state' +sleep 2 +curl -sS -D - -o /dev/null "$URL" | grep -iE 'x-cache|age:|x-served-by|hit-state' +``` + +Expected on the second request: if a cache sits in front of the Compute service, an `age` +greater than zero or an `x-cache` containing `HIT`. + +**The probe above is weak evidence** — absence of `age` is equally consistent with "no +cache" and "cold cache". **The topology check below is the actual answer; run it first and +skip the probe if it is conclusive.** + +```bash +fastly service list +fastly service-version list --service-id +# Look for a Delivery service fronting the Compute service, and for shielding +# configured on the service rather than only on the origin backend. +``` + +A Compute service with no Delivery service in front and no fronting shield does not have +its own output cached — that is the configuration the spec assumes, and this step exists +to confirm or refute it rather than to leave it assumed. + +**While you have the service open, answer a second question that matters more than this +task does:** is the _publisher backend_ shielded on the TS service? + +```bash +fastly backend list --service-id --version active +# Look for a shield on the publisher origin backend. +``` + +#1009's entire off-TS advantage came from a **shield** HIT, not a POP HIT. Whether +Stage 0 recovers a shield HIT or only a single-POP HIT changes the size of the win +materially, and nothing else in this plan establishes it. + +- [ ] **Step 3: Record the finding** + +Append to the findings document: + +```markdown +## Step B — consumers of TS's own response headers + +**Verdict:** SHARED CACHE PRESENT / NO SHARED CACHE + +**Evidence:** + +**Consequence:** NO SHARED CACHE → spec Stage 3b is inert until a topology change; +deprioritize it and ship only Stage 3a (browser caching). SHARED CACHE PRESENT → +Stage 3b gains a consumer AND the per-user `x-geo-*` header leak in spec §7 becomes an +active privacy exposure rather than a theoretical one. Escalate immediately in that case. +``` + +- [ ] **Step 4: Commit** + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record response-header cache consumer findings for #1009" +``` + +--- + +## Task 3: Step C — origin fetch timing, and the bypass flag + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/settings.rs` — `publisher.bypass_origin_cache`, + `default_bypass_origin_cache`, `debug.publisher_timing`, the `Publisher` `Default` impl, + eight test literals, and the `origin_host` doctest +- Modify: `crates/trusted-server-core/src/test_support.rs` (log-capture helper) +- Test: inside `mod ssat_cache_policy_tests` at `crates/trusted-server-core/src/publisher.rs:4541` + +**Use `web_time::Instant`, not `std::time::Instant`** — the workspace targets +`wasm32-wasip1` and `web_time` is the wasm-safe clock already used at +`crates/trusted-server-core/src/auction/orchestrator.rs:7`. + +### Two timings, and why these two + +Measure **`hold_wait_ms`** and **`origin_fetch_ms`**. Not the rewrite. + +`hold_wait_ms` is the decision. The hold's cost is literally the duration of one +`.await` — `collect_stream_auction` at `publisher.rs:793`, plus the two EOF variants in +`hold_finish_ready_segments` (`:869`) and `hold_finish_tail_segments` (`:896`). Two +`Instant`s around those calls answer "does the hold block?" directly, instead of +inferring it by comparing origin fetch against auction duration. + +`origin_fetch_ms` is attribution — how much of any win Stage 0 can claim. + +`rewrite_ms` decides nothing. Step C's verdict compares origin fetch against auction +collect, and the ceiling argument in spec §6.4 is structural — it needs no number. +Measuring the rewrite would mean instrumenting two finalizers +(`buffer_publisher_response_async` at `publisher.rs:1114`, and the +`async_stream::try_stream!` block at `publisher.rs:1286`), working around moves out of +`params` inside that block, and finding a correlation key that does not exist — +`OwnedProcessResponseParams` (`publisher.rs:1065-1087`) has no `request_path`, and adding +one means touching all 26 construction sites. + +None of that buys a decision. Skip it. If a rewrite figure is later wanted to set a +target, add it as a separate follow-on once the verdict is known. + +**Why a log line and not `Server-Timing`:** for `origin_fetch_ms` alone a response header +would in fact work — the value is known before headers commit. A log line is still +preferred because it is server-side (no dependence on a browser harness to collect it), +`log` is this project's instrumentation crate per `CLAUDE.md`, and the auction path +already measures itself the same way. The spec previously claimed `Server-Timing` cannot +work at all; that overbroad claim has already been corrected there. + +### Log volume — gate it + +The line sits after the origin send, so it fires for every publisher request that reaches +origin — tagged `ad_stack=false` for ineligible ones, not only for eligible navigations. +That is more useful for comparison and more log spend, and the instrumentation is +temporary either way. Gate it behind the existing debug surface rather than +emitting unconditionally: add a `#[serde(default)] pub publisher_timing: bool` to +`DebugConfig` (`crates/trusted-server-core/src/settings.rs:1872`), following +`ja4_endpoint_enabled` and `auction_html_comment` alongside it. Default `false`; enable +via `ts config push` for the measurement window, then disable. + +This also means the Step 1 test must set that flag in its settings fixture. + +The split is also what makes the Step 1 test achievable — `run_with_slots` +(`publisher.rs:4769`) invokes only `handle_publisher_request` and never drives either +finalizer, so a test asserting on a combined line could never pass. + +**What `origin_fetch_ms` actually measures.** `publisher.rs:2863-2865` sets +`.with_stream_response()` when the adapter supports it, so on Fastly `send()` returns at +response _headers_, not after the body downloads. `origin_fetch_ms` is therefore **origin +TTFB**, not full download time. Name it that way in the findings document. It is still +the correct before/after signal for Stage 0 — the bypass affects whether the request hits +a cache at all — but when comparing against auction `total_time_ms` in Step 9, compare +like with like and say which quantity each column holds. + +- [ ] **Step 1: Write the failing test** + +**Placement matters.** Add the test **inside `mod ssat_cache_policy_tests`** +(`publisher.rs:4541`), not the outer `mod tests` (`:4035`). Every helper it uses is +private to that nested module: `settings_with_enabled_auction_and_creative_opportunities` +(`:4684`), `article_slot` (`:4721`), `conditional_navigation_request` (`:4740`), +`queue_cacheable_html_response` (`:4752`), `run_with_slots` (`:4769`). Placed in the outer +module it will not resolve — and because two _other_ `article_slot` functions exist +(`:9593`, `:10276`) returning a different type, the failure surfaces as a confusing type +error rather than a missing-name error. + +**First, add the log-capture helper.** `crates/trusted-server-core/src/test_support.rs` +has none. Note its shape: the whole file is `#[cfg(test)] pub mod tests { … }`, so the +path is `crate::test_support::tests::capture_logs`, not `crate::test_support::capture_logs` +— see existing consumers at `auth.rs:103` and `config_payload.rs:48`. + +Two constraints the helper must respect or the test fails for unrelated reasons: + +- `log::set_boxed_logger` succeeds **once per process**. Install via a `OnceLock`/`Once` + and have `capture_logs()` return a guard that clears and then reads a shared buffer. +- Call `log::set_max_level(log::LevelFilter::Info)` or higher, or `log::info!` is filtered + out before it reaches the logger. +- **Do not have the guard hold the buffer's own `Mutex`.** The test body runs code that + calls `log::info!` on the same thread, and the logger must lock that same mutex to + append — `std::sync::Mutex` is not reentrant, so this **hangs** rather than failing. + Use two locks: a separate process-wide serialization mutex held by the guard, and the + buffer's own mutex taken and released per line by the logger. +- The buffer is process-global and every other concurrently-running `trusted-server-core` + test logs into it, so a `got: {captured}` diagnostic will be large. Assert with + `contains`, not equality. +- `log::set_max_level` is global for the test binary. Setting it to `Info` is fine, but it + affects every test in the process. + +```rust +#[tokio::test] +async fn eligible_navigation_logs_origin_fetch_duration() { + // Arrange + let logs = crate::test_support::tests::capture_logs(); + let mut settings = settings_with_enabled_auction_and_creative_opportunities(); + // The log line is gated; without this the assertions below can never pass. + settings.debug.publisher_timing = true; + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let _ = run_with_slots(&settings, &services, &slots, conditional_navigation_request()).await; + + // Assert + let captured = logs.contents(); + assert!( + captured.contains("publisher_timing"), + "eligible navigation should emit a publisher_timing log line, got: {captured}" + ); + assert!( + captured.contains("origin_fetch_ms="), + "publisher_timing should record origin_fetch_ms, got: {captured}" + ); +} +``` + +This test deliberately asserts only on the `publisher_timing` line. `run_with_slots` never +drives a finalizer, so `publisher_rewrite` is out of its reach — cover that separately if +at all, rather than contorting this test. + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation_logs_origin_fetch_duration -- --nocapture +``` + +Expected: FAIL — no `publisher_timing` in the captured logs. (Substitute your host +triple; core tests run natively for fast iteration. The Viceroy run comes in Step 6.) + +- [ ] **Step 3: Time the origin fetch** + +In `publisher.rs`, at the top with the other imports, add: + +```rust +use web_time::Instant; +``` + +Then wrap the origin send. The current code is at `publisher.rs:2870`: + +```rust +let mut response = match services.http_client().send(platform_request).await { +``` + +Change it to: + +```rust +let origin_fetch_start = Instant::now(); +let mut response = match services.http_client().send(platform_request).await { +``` + +and immediately after the `match` completes (after the existing `};` that closes it, +before the existing `log::debug!("Publisher origin response received: ...")` at `:2888`): + +```rust +let origin_fetch_ms = u64::try_from(origin_fetch_start.elapsed().as_millis()).unwrap_or(u64::MAX); +``` + +**Make the bypass config-driven in the same change.** This is what lets Stage 0 ship as a +config flip rather than a second deploy — see Task 5. Replace the block at +`publisher.rs:2866-2868`: + +```rust +// Single source of truth for the request and the log line below. Operator- +// controlled so the read-through cache can be re-enabled without a release; +// see docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md §4. +let cache_bypass = should_run_ad_stack && settings.publisher.bypass_origin_cache; +if cache_bypass { + platform_request = platform_request.with_cache_bypass(); +} +``` + +Add the setting to `Publisher` in `crates/trusted-server-core/src/settings.rs:29`, +**defaulting to today's behaviour** so this change is a no-op until deliberately flipped: + +```rust +/// Bypass the platform read-through cache on ad-eligible publisher navigations. +/// +/// `true` preserves the historical behaviour introduced by the SSAT 304-prevention +/// design. `false` lets those navigations use the read-through cache; the +/// conditional-header strip already guarantees a complete body on a cache HIT. +/// Temporary operator control for the Stage 0 rollout — remove once settled. +#[serde(default = "default_bypass_origin_cache")] +pub bypass_origin_cache: bool, +``` + +```rust +fn default_bypass_origin_cache() -> bool { + true +} +``` + +**Adding this field breaks nine sites. Update them in the same commit or Step 2 fails to +compile before it can produce the intended RED failure:** + +- The hand-written `Default` impl at `settings.rs:81-97`. +- Eight exhaustive test literals. The line numbers below anchor each + `let publisher = Publisher {` **opening**, not a field — add the new field inside each + brace: `settings.rs:3553`, `:3564`, `:3575`, `:3586`, `:3597`, `:3608`, `:3621`, + `:3635`. `clippy-fastly` runs `--all-targets`, so these gate lint too. +- The rustdoc example for `origin_host`, whose literal opens at `settings.rs:130`. + **This is a live doctest** and the host-triple test command below does not skip + doctests. + +While there, mirror the existing default-agreement test +`publisher_default_max_buffered_body_bytes_matches_config_default` (`settings.rs:3648`) — +it exists to catch a hand-written `Default` diverging from a serde default, which is +exactly the shape this field re-introduces. One assertion. + +Then emit the line, immediately after computing `origin_fetch_ms`, gated on the debug +flag from the section above: + +```rust +if settings.debug.publisher_timing { + log::info!( + "publisher_timing origin_fetch_ms={origin_fetch_ms} \ + cache_bypass={cache_bypass} ad_stack={should_run_ad_stack}" + ); +} +``` + +- [ ] **Step 4: Instrument `hold_wait_ms` — the decision metric** + +This is the number the whole effort turns on, and it needs **one edit in one function**. + +`collect_stream_auction` (`publisher.rs:2431`) is the only function that awaits the +auction collect, and all three call sites reach it: + +| Call site | Path | +| ------------------- | ------------------------------------------------------------ | +| `publisher.rs:793` | `hold_collect_close_tail` — Fastly lazy stream | +| `publisher.rs:2257` | `body_close_hold_loop`, EOF arm — Axum, Cloudflare, Spin | +| `publisher.rs:2311` | `body_close_hold_loop`, mid-stream arm — same three adapters | + +Instrument the callee, not the callers. It already destructures `settings` out of +`AuctionCollectDeps` (`:2436`), so the debug flag is in scope with no new plumbing, and +one edit covers every adapter. + +Wrap the `collect_dispatched_auction` await at `:2447-2449`: + +```rust + let hold_wait_start = Instant::now(); + let result = orchestrator + .collect_dispatched_auction(dispatched, services, &collect_ctx) + .await; + if settings.debug.publisher_timing { + let hold_wait_ms = + u64::try_from(hold_wait_start.elapsed().as_millis()).unwrap_or(u64::MAX); + log::info!("publisher_hold hold_wait_ms={hold_wait_ms}"); + } +``` + +`settings` here is `&&Settings` from the destructure — deref as needed; the compiler will +say so. + +**Do not instrument `hold_finish_ready_segments` (`:869`) or `hold_finish_tail_segments` +(`:896`).** Neither awaits the collect. The first returns `close_found` for its caller to +act on; the second delegates to `hold_collect_close_tail` at `:909`. Instrumenting them +would double-count. + +**Do not instrument the auction itself.** `OrchestrationResult::total_time_ms` +(`orchestrator.rs:285`, struct at `:1449`, per-provider at `:365`) already flows to +`auction_events_raw`. `hold_wait_ms` measures something different and more useful: how +long the _response_ waited, which is near zero when the auction finished during transfer +even though `total_time_ms` is large. + +- [ ] **Step 5: Run the test to verify it passes** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation_logs_origin_fetch_duration -- --nocapture +``` + +Expected: PASS. + +- [ ] **Step 6: Run the full publisher test module under the real target** + +A format-changing edit to this file can break tests far from the one you added, and the +Viceroy runner aborts on the first panic — so run the whole suite, not a filtered subset. + +```bash +cargo test-fastly +``` + +Expected: PASS. `app::tests` DNS `Error` lines in the output are pre-existing noise. + +- [ ] **Step 7: Verify format and lint** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly +``` + +Expected: both clean. + +- [ ] **Step 8: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-core/src/settings.rs \ + crates/trusted-server-core/src/test_support.rs +git commit -m "Add an operator switch for the origin cache bypass and log origin fetch time" +``` + +Staging without `settings.rs` leaves a tree that does not compile. + +- [ ] **Step 9: Deploy and collect** + +Deploy first. Then enable the log — it is gated and off by default: + +```bash +# In the operator's trusted-server.toml, under [debug]: +# publisher_timing = true +ts config push +``` + +**Deploy before pushing, not after.** `Settings`, `Publisher`, and `DebugConfig` all carry +`#[serde(deny_unknown_fields)]`, and `ts config push` validates against the typed schema +(`crates/trusted-server-cli` → `run_config_push_typed::`). So the +`ts` binary must be rebuilt from this commit (`cargo install-cli`), and pushing the new +keys before the new WASM is live would break config load on the deployed build. +`trusted-server.example.toml:121-125` records this same hazard for +`auction.rewrite_creatives`. + +Then capture the **bypass-on baseline only**. Do not try to collect an off arm here — +turning the bypass off _is_ Task 5, which is gated on Task 1's verdict and forbidden on a +FAIL. The off arm is collected in Task 5 Step 8. + +Capture enough navigations to separate the medians with confidence, across both a homepage and an article path, with the bypass +both on and off. Record the N alongside the result. + +Append to the findings document: + +```markdown +## Step C — server-side latency breakdown + +**N per arm:** · **Paths:** · **Date:** + +| Arm | `origin_fetch_ms` = origin TTFB (median) | auction `total_time_ms` (median) | `rewrite_ms` (median) | +| ---------- | ---------------------------------------- | -------------------------------- | --------------------- | +| bypass on | | | | +| bypass off | | | | + +Read the asymmetry carefully. `origin_fetch_ms` is origin **TTFB** — the send returns at +response headers because `.with_stream_response()` is set — whereas `total_time_ms` is +the auction's full duration. The comparison below is still the right one, but it is not +comparing two like quantities. + +**Verdict:** HOLD IS FREE / HOLD IS COSTING + +Read it off `hold_wait_ms` directly — no model, no comparison against auction duration. + +HOLD IS FREE = `hold_wait_ms` median near zero. The auction finishes during body +transfer. Proceed as staged in spec §7: Stage 0 primary, Stage 2 protects its win. + +HOLD IS COSTING = `hold_wait_ms` median materially non-zero. **Staging inverts** — +Stage 2 becomes primary and Stage 0 secondary. The work does not change, only its order. +Spec §6.2 argues for the first outcome but explicitly does not prove it, so treat the +second as a live possibility. +``` + +- [ ] **Step 10: Commit the findings** + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record server-side latency breakdown for #1009" +``` + +--- + +> **Task 4 (spec correction) was completed while this plan was being written.** §3's +> mechanism bullet, §4's operator-flag framing, and the `unexpected_origin_304` watch are +> all already in the spec. Nothing to do; the task is removed rather than left as a +> no-op an implementer would stall on. + +--- + +## Task 5: Stage 0 — turn the origin cache bypass off + +**Gate:** do not flip the flag until Task 1 has recorded a **`FINAL PASS`**. There are +three verdicts, not two. + +- **`FINAL PASS`** → Task 5a (config flip). +- **`PROVISIONAL PASS`** → **stop.** Not a release gate. This is the current state. It + means the representation split is declared correctly under the conditions tested, and + that those conditions were too narrow to flip production on. +- **`FAIL`** → Task 5b. Do **not** flip; it can serve an RSC payload to an HTML + navigation. + +**`FINAL PASS` requires all five, each recorded in the findings document:** + +| Condition | Why the provisional run is insufficient | +| -------------------------------------------------------- | ---------------------------------------------------- | +| A real authenticated or state-bearing session cookie | `sessionid=abc123` is synthetic and proves nothing | +| Basic Auth exercised **through TS**, not just the origin | #1009 describes a gated deployment | +| The experiment variant named in #1009 | Absent from the origin's `Vary`; unexplained | +| Representative routes — article, section, search | Only the homepage was probed | +| Cached-hit slot and render attribution | The randomized div IDs are an unverified interaction | + +Any one of these unrecorded means the verdict stays `PROVISIONAL PASS` and Task 5 does +not start. + +Because Task 3 made the bypass config-driven, Stage 0 ships as a **config change on an +already-deployed build** — no second release, and the read path reverts with another +config push rather than a revert. That matters here: the failure mode this gates on is +cache poisoning, where minutes of exposure are worse than a slow rollout. + +**But a config push is not a full rollback.** It stops HTML navigations reading from +cache; it evicts nothing already stored. See Step 4's rollback sequence — flip, then purge +or roll a versioned namespace, then observe past the origin TTL. Until a C1 purge path +exists, the tail is "wait out the origin TTL," and that must be an accepted, recorded +risk before the flip. + +### Task 5a: flip the flag (Task 1 verdict = `FINAL PASS`) + +**Files:** + +- Modify: the operator's `trusted-server.toml` (gitignored) +- Modify: `crates/trusted-server-core/src/publisher.rs` — the test, and later the default +- Modify: `trusted-server.example.toml` — document the key + +- [ ] **Step 1: Add a test covering the flag in both positions** + +The existing test at `publisher.rs:4824` +(`eligible_navigation_bypasses_cache_and_returns_non_storable_html`) asserts `vec![true]` +and must **keep passing** while the default is `true` — it now documents the default +rather than the only behaviour. Leave it, and add a sibling next to it: + +```rust +#[tokio::test] +async fn eligible_navigation_uses_read_through_cache_when_bypass_disabled() { + // Arrange + let mut settings = settings_with_enabled_auction_and_creative_opportunities(); + settings.publisher.bypass_origin_cache = false; + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let response = + run_with_slots(&settings, &services, &slots, conditional_navigation_request()).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "disabling bypass_origin_cache should let the navigation use the read-through \ + cache; the conditional-header strip already guarantees a full body on a HIT" + ); + assert_eq!( + recorded_header( + stub.recorded_request_headers().first().expect("should record request"), + header::IF_NONE_MATCH.as_str() + ), + None, + "conditional headers must still be stripped with the bypass disabled" + ); + assert!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("no-store")), + "the synthesized document must stay non-storable regardless of the bypass flag" + ); +} +``` + +Those last two assertions are the point of the test: the flag must change **only** the +cache mode, leaving the conditional-header strip and the response non-storability intact. + +**Leave `publisher.rs:4941` and `:5160` unchanged** — they already assert `vec![false]` +for non-eligible requests and must keep doing so. `Range`/`If-Range` stripping is covered +by `eligible_range_navigation_fetches_complete_html` (`publisher.rs:4883`), unaffected. + +- [ ] **Step 2: Run both tests** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation -- --nocapture +``` + +Expected: both the existing default-behaviour test and the new flag-disabled test PASS. +If Task 3's `bypass_origin_cache` field is not yet in place, the new test will not +compile — land Task 3 first. + +- [ ] **Step 3: Document both keys in the example config** + +Add to `trusted-server.example.toml` under `[debug]` (line 149, alongside +`ja4_endpoint_enabled` and `auction_html_comment`): + +```toml +# Emit a `publisher_timing` log line per publisher origin fetch. Temporary +# instrumentation for the #1009 latency measurement; leave false in production. +publisher_timing = false +``` + +And under `[publisher]`: + +```toml +# Bypass the platform read-through cache on ad-eligible navigations. +# `true` is the historical default. Set `false` to let those navigations use the +# read-through cache — only after confirming the origin declares `Vary` for every +# header it varies on (see the Stage 0 precondition). +bypass_origin_cache = true +``` + +- [ ] **Step 4: Flip it in the operator config and push** + +```bash +# In the operator's trusted-server.toml, under [publisher]: +# bypass_origin_cache = false +ts config push +``` + +Note from prior operational experience in this repo: the environment-variable overlay is +scalar-only **and** only overrides keys that already exist in the TOML. Adding the key to +the operator's file is required; setting only an env var will be silently dropped. + +**Rollback is a config push plus an eviction — not a config push alone.** Pushing `true` +again stops HTML navigations reading from cache, but evicts nothing: objects already +cached, including those RSC and other request classes keep reading, persist until they +expire. The origin's `max-age=60` bounds that, but does not remove it. + +Full rollback: + +1. Push `bypass_origin_cache = true`. +2. Purge — **and note this is C1, not C2.** `InsertBuilder::surrogate_keys` belongs to + the Core Cache API and applies to the transformed-template cache the ESI spike builds. + It has no effect on the HTTP read-through cache that Stage 0 turns on. Purging C1 + requires either surrogate keys the **origin** supplies on its responses, or the HTTP + cache's own request/candidate surrogate-key surface. Confirm which is available before + relying on it. + + **Neither is wired today.** If the flip ships before one exists, the rollback story is + "wait out the origin TTL" — roughly a minute, per the Step A findings. That is + survivable, but it must be an accepted risk recorded before the flip rather than a + discovery during an incident. + +3. Observe past the origin TTL before declaring the incident closed. + +- [ ] **Step 5: Run the full suite across every adapter** + +```bash +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +``` + +Expected: all PASS. If `platform/test_support.rs:797` or `:888` fail, they are testing +the stub's own recording behaviour rather than publisher behaviour — read them before +changing anything. + +- [ ] **Step 6: Format and lint every target** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare \ + && cargo clippy-cloudflare-wasm && cargo clippy-spin-native && cargo clippy-spin-wasm +``` + +Expected: all clean. + +- [ ] **Step 7: Commit the code and config-template changes** + +```bash +git add crates/trusted-server-core/src/publisher.rs trusted-server.example.toml +git commit -m "Add an operator switch for the publisher origin cache bypass" +``` + +- [ ] **Step 8: Watch for the failure modes, not just the win** + +After the flip, check three things before declaring success. The first two are regression +signals, not confirmations. + +1. **`unexpected_origin_304` abandonment telemetry.** This reason + (`publisher.rs:2896`, emitted via `emit_abandoned_auction` at `:2360`) exists because + the ad-stack path refuses cached and conditional origin responses. Re-enabling the + cache is precisely what could revive it. **Any non-zero rate is a rollback signal** — + it means a 304 is reaching TS, which the conditional-header strip was supposed to make + impossible. Push `true` and investigate before continuing. +2. **Representation mixing.** Spot-check that HTML navigations still return HTML and RSC + fetches still return `text/x-component`. A mismatch is the Task 1 risk having + materialized despite a PASS verdict — roll back immediately, this is cache poisoning. +3. **`origin_fetch_ms` and `cache_bypass=false`** in the `publisher_timing` logs. This is + the win, and it is the _last_ thing to check, not the first. + +- [ ] **Step 9: Record and commit** + +Append the before/after medians and the three checks above to the findings document, +format it, and commit. + +- [ ] **Step 10: Retire the flag (follow-up, not now)** + +Once the flip has held for a sustained period, flip the default to `false` in +`default_bypass_origin_cache`, then remove the setting and the branch entirely. Track it; +a temporary flag left in place becomes permanent configuration surface. + +### Task 5b: cache-key discriminator (Task 1 verdict = FAIL) + +**Do not implement from this plan.** A FAIL means the origin serves multiple +representations at one URL without declaring `Vary`, so removing the bypass requires TS +to add its own cache-key discriminator — a feature, not a deletion, and materially larger +than Stage 0 as scoped here. + +Escalate with the Task 1 findings and write a separate plan. Two things that plan must +address, both from spec §4: + +1. The discriminator must key on the request headers that actually distinguish the + representations (`RSC`, `Next-Router-*`, the experiment header), **not** on the + navigation classification. `is_navigation_request` + (`crates/trusted-server-core/src/http_util.rs:73-98`) falls back to the `Accept` + header when Fetch Metadata is absent, and its own comment warns that `fetch()` can set + `Accept: text/html` — so a fetch-based request can be misclassified as a navigation. +2. Whether the origin should simply be asked to declare `Vary`, which is cheaper than + building the discriminator and fixes the problem for every consumer rather than only + for TS. + +--- + +## Out of scope + +Named so nobody widens this plan mid-flight. All are specified in the spec. + +- **Stages 1–2** — moving bid delivery off the response body and deleting the `` + hold. Spec §7 and §8 put these behind the correctness defects. Spec §5 explains why + starting them casually produces a silent revenue loss. +- **Stages 3a/3b** — response cacheability. 3b is additionally gated on Task 2. +- **Stages 4–5** — purge capability, TS-owned template cache, ESI. +- **Removing the `bypass_cache` platform capability.** Task 5a removes one call site only. + +--- + +## Definition of done + +- [ ] Findings document records verdicts for Steps A, B, and C, each with its date, its + N where applicable, and the consequence spelled out. +- [ ] `publisher_timing` and `publisher_hold` lines are emitted in production and + readable, and `hold_wait_ms` has a recorded median. +- [ ] Task 1 recorded a **`FINAL PASS`** — all five conditions in Task 5's gate closed, + not merely the provisional run. +- [ ] Either Task 5a is shipped, or Task 1 returned FAIL and both a production defect and + a follow-up plan for 5b exist. +- [ ] A purge path or versioned cache-key namespace exists **before** the flip, or the + "wait out the TTL" rollback is explicitly accepted and recorded as a risk. +- [ ] **The win is measured client-side, not from `origin_fetch_ms`.** That figure is + origin TTFB and excludes body download, rewrite, and post-processing — it is + attribution, not the outcome. #1009 already has a working tester-cookie browser A/B + measuring the TTFB the publisher actually complained about; use it for before/after. +- [ ] `unexpected_origin_304` rate is zero and representations are not mixed (Task 5a + Step 8) — both checked **before** the win is claimed. +- [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four + adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md new file mode 100644 index 000000000..fa3e90544 --- /dev/null +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -0,0 +1,503 @@ +# #1009 measurement findings + +Recorded output of the checks in +[the plan](./2026-08-08-1009-measurement-and-stage-0.md). Results only — the origin +hostname is operator config and is deliberately not reproduced here. + +> **Final implementation note, 2026-08-12.** The opt-in `esi` mode now uses Fastly +> Core Cache plus an exact inert byte seam. The parser and client-fill experiments were +> removed. Real-origin numbers in this file remain evidence about the observed path, not +> a clean before/after benchmark. Current operational semantics are documented in +> [the configuration guide](../../guide/configuration.md). + +## Step A — origin `Vary` declaration and cookie exposure + +**Date:** 2026-08-08 · **Method:** direct `curl` against the publisher origin with the +configured `origin_host_header_override`, homepage path. + +### Representation split + +| Representation | `Content-Type` | `Cache-Control` | `Set-Cookie` | +| ------------------------------------ | ------------------ | --------------- | ------------ | +| HTML navigation | `text/html` | `max-age=60` | none | +| `RSC: 1` | `text/x-component` | `max-age=60` | none | +| `RSC: 1` + `Next-Router-Prefetch: 1` | `text/x-component` | `max-age=60` | none | + +`Vary`, identical on every response: + +``` +vary: rsc, next-router-state-tree, next-router-prefetch, next-router-segment-prefetch, Accept-Encoding +``` + +The origin declares **every** header that distinguishes the representations, including +`next-router-segment-prefetch`, which the plan's probe list did not think to check. The +HTML/RSC split at one URL is real and correctly declared. + +### Cookie personalization + +Hash comparison was useless here and the plan's probe as written would have produced a +false FAIL — see the method note below. After normalizing per-request identifiers: + +| Comparison | Differing lines | +| ------------------------------ | --------------- | +| no-cookie A vs no-cookie B | 2 | +| no-cookie A vs **with cookie** | 2 | + +Both diffs are the same single `generationTimestamp` field in the RSC payload. **The +cookie changes nothing.** Byte lengths were identical across all three responses +(1,432,944). + +Cookie sent: `ts-tester=true; sessionid=abc123; ts-ec=probe`. + +### Verdict: **PROVISIONAL PASS** — not sufficient to gate a production flip + +Downgraded 2026-08-10 after external review. Everything below held under the conditions +tested; the conditions tested are narrower than the gate requires. + +What passed: + +- `Vary` names every request header the origin varies on. ✅ +- Bodies did not differ by the cookie sent, so `Vary: Cookie` was not required **for + that cookie**. ✅ +- No `Set-Cookie` on a shared-cacheable response. ✅ +- Origin returns 200 without credentials at this layer. ✅ + +**What was not tested, and each of these can flip the verdict:** + +| Gap | Why it matters | +| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `sessionid=abc123` is not a real session | A synthetic value proves nothing about a state-bearing publisher session. An authenticated or paywall-metered session is exactly the case that would personalize. | +| One route (homepage) only | Article, section, and search routes may personalize differently. | +| Experiment variant never exercised | #1009 says the origin varies on one. It is absent from `Vary` — see Residual uncertainty below. | +| Basic Auth through TS untested | #1009 describes a gated deployment. Only the origin was probed directly. | +| Cached-hit slot resolution untested | The randomized div IDs below are an unverified interaction, not a cleared one. | + +**Consequence:** Stage 0 still takes the operator-flag path rather than the cache-key +discriminator, and no live cross-serving defect is indicated. But this is **not** a +release gate. Close the table above before flipping the flag in production. + +## Two findings the checks were not looking for + +### 1. The origin already intends this page to be shared-cached + +`cache-control: max-age=60` with a correct `Vary` and no `Set-Cookie`. The origin has +been cacheable all along; Trusted Server opted out of it. That is the spec's §4 framing +confirmed from the other side, and it strengthens the case that the bypass was +belt-and-braces rather than load-bearing. + +It also bounds the win: a 60-second TTL means Stage 0 buys a cache hit only within that +window. Whether that translates into a meaningful hit rate depends on request volume per +URL, which is not measured here. + +### 2. Ad-slot div IDs are randomized per request — and this interacts with Stage 0 + +The only per-request variance in the document is ~170 lines of ad-slot container IDs, +each a fresh 32-hex UUID: + +``` +ad-in_content-f75fa7fba54a4fc2a2d787f51c1837dd-in_content-0 +ad-in_content-a968b27e3ee2424f8bb1c19560abf2b1-in_content-0 ← same slot, next request +``` + +Under the bypass, Trusted Server sees fresh IDs on every request. **Once the cache is on, +every visitor within a 60-second window receives the same IDs.** + +This is very likely fine — `tsjs.adSlots` is built from configured slot definitions, not +scraped from origin markup, and injection is a prefix match on the configured `div_id`. +But it is an untested interaction between Stage 0 and the slot-matching path, and it was +not in anyone's risk list. **This is a release gate, not a note.** Verify slot matching resolves against a cached +document before flipping the flag, and watch TS-attributed renders across the flip +rather than only `origin_fetch_ms`. + +## Method note — a defect in the plan's Step A probe + +The plan's cookie check compares `shasum` of the response bodies. On this origin that +test always fails, cookie or not, because of the randomized div IDs above. Three requests +produced three different hashes with byte-identical lengths. + +**Correct method:** normalize per-request identifiers before comparing, e.g. +`sed -E 's/[0-9a-f]{32}/UUID/g'`, and diff the normalized bodies rather than hashing +them. Establish the no-cookie baseline drift first, then compare the cookie arm against +that baseline — a cookie arm is only interesting if it differs by _more_ than the +baseline does. Fix the plan before anyone re-runs this. + +## Residual uncertainty + +#1009 states the origin varies on an experiment header as well as `rsc` and +`next-router-*`. **No experiment header appears in the origin's `Vary` list**, and the +RSC payload's `experiments` key did not differ across any of the requests made here. + +Three readings, unresolved: the issue was imprecise; experiments are assigned +client-side; or they key on a cookie value this probe did not supply. The `Vary` +declaration is authoritative for cache correctness and it is thorough enough to name four +Next-specific headers, so this is unlikely to be a cache-safety gap. Worth one question +to whoever wrote that line in #1009 rather than further probing. + +## Rollback caveat, added 2026-08-10 + +The plan described flipping the flag back as a seconds-long rollback. That is +incomplete. Re-enabling the bypass stops **HTML navigations** reading from cache; it +evicts nothing. Objects already cached — including those RSC and other request classes +continue to read — persist until they expire. + +Two mitigations, both real: + +- The origin's `max-age=60` bounds read-through exposure to roughly a minute. +- Purge exists in-process — `fastly::http::purge::purge_surrogate_key`. An earlier claim + that TS had no purge capability was wrong; it has no _wiring_, which is buildable. + +**But note which cache.** `InsertBuilder::surrogate_keys` belongs to the **Core Cache** +API and applies to the transformed-template cache the ESI spike would build (C2). It has +**no effect on the HTTP read-through cache** that Stage 0 turns on (C1). Purging C1 needs +surrogate keys the _origin_ supplies on its responses, or the HTTP cache's own +request/candidate surrogate-key surface. Confirm which is available before relying on it — +an earlier revision of this document conflated the two. + +**C2's purge is locally testable; C1's is not.** Verified 2026-08-10: Viceroy 0.17 +implements `purge_surrogate_key` against the same in-process cache it serves reads from +(`viceroy-lib-0.17.0/src/wiggle_abi/fastly_purge_impl.rs:10-32`), soft purge included. So +the purge-based rollback for the C2 template cache the spike builds can be exercised end +to end without a Fastly service. That does nothing for Stage 0, whose exposure is C1. + +Rollback is therefore: flip the flag, **then** purge C1 by whichever mechanism is actually +available (or roll a versioned key namespace), **then** observe past the origin TTL before +declaring the incident closed. With no C1 purge path wired, the tail is the TTL itself — +roughly a minute here, and a recorded risk rather than a surprise. + +## ESI spike Task 1 — does `esi` 0.7 build on this toolchain? + +**Date:** 2026-08-10 · **Verdict: PASS.** The cheapest falsifier for the ESI question +clears. #1009 is not closed by a toolchain limit. + +| Check | Result | +| ------------------------------------------------------------------------------------------ | -------------------- | +| `cargo add esi@0.7 --package trusted-server-adapter-fastly` | resolved `esi 0.7.1` | +| `cargo check-fastly` (Rust 1.95.0 / `wasm32-wasip1`) | clean | +| `cargo fmt --all -- --check` | clean | +| All six clippy targets (fastly, axum, cloudflare, cloudflare-wasm, spin-native, spin-wasm) | clean | +| `cargo check --manifest-path crates/trusted-server-integration-tests/Cargo.toml --tests` | clean | + +**Nine new transitive dependencies:** `esi 0.7.1`, `nom 8.0.0`, `rand 0.10.2`, +`rand_core 0.10.1`, `chacha20 0.10.1`, `cpufeatures 0.3.0`, `atoi 2.0.0`, +`html-escape 0.2.15`, `md5 0.8.1`. + +**No existing shared dependency moved.** `regex` stays 1.12.4, `bytes` 1.12.0, `log` +0.4.33. `nom` and `rand` gain new majors that coexist with the existing 7.1.3 / 0.8.6 / +0.9.4 rather than replacing them — the best available outcome, since a forced bump on a +shared dep is what would have made this expensive. + +### A claim in the spike plan was wrong + +Task 1 Step 3 told the implementer to check for a desync between the root `Cargo.lock` and +`crates/trusted-server-integration-tests/Cargo.lock`. **That second lockfile does not +exist.** The integration-tests crate is a workspace member (root `Cargo.toml:10`) and +shares the root lockfile, so the desync hazard cannot arise in that form. The plan has +been corrected. The dual-lockfile constraint was real at some earlier point; it is not the +current layout. + +### Viceroy 0.17 supports the whole Core Cache surface this spike needs + +**Date:** 2026-08-10 · **Verdict: PASS.** Probed directly under +`cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1`, then removed: + +| API | Result | +| ------------------------------------------------------------------------ | ------ | +| `cache::core::insert(key, ttl).execute()` → write → `finish()` | works | +| `cache::core::lookup(key).execute()` → `Found::to_stream()` | works | +| `Transaction::lookup(key).execute()` → `must_insert_or_update()` | works | +| `Transaction::insert(ttl).surrogate_keys([…]).execute_and_stream_back()` | works | +| Second transactional lookup reports a hit, no obligation | works | + +That is the entire API surface the spike's Task 3 Step 4 specifies, including the +transaction and stream-back shapes. + +**Consequence: provisioning a Fastly service is not a prerequisite.** An earlier revision +of the spike plan made it Task 2 and a blocker on everything downstream. Almost all of the +correctness and safety work — the C2 cache logic, the transform, template byte-identity, +ESI assembly, DCA and dispatcher refusal, fragment-failure degradation, header ordering, +and the leakage gates — runs locally. The plan is re-sequenced accordingly. + +**What still needs a real service:** shielding behaviour, POP-level cache tiering, +request collapsing under genuine concurrency (Viceroy is a single instance, so a passing +`Transaction` test proves the API works and not that collapsing is correct under load), +stale revalidation timing, and **every performance number in the decision rule**. Local +timings are meaningless for the decision. + +### Not yet verified + +Compiling and a cache round-trip are not an implementation. Nothing yet exercises the +`lol_html` transform into C2, ESI assembly, or any runtime behaviour on the publisher path, +and the `esi` dependency is added but unused. + +## ESI spike Task 3 — implementation progress + +**Date:** 2026-08-10. All of it behaviour-neutral under the default +`AssemblyMode::Inline`; nothing here changes a shipped code path. + +| Step | State | +| -------------------------------- | ------------------------------------------------------------------- | +| 1 — `AssemblyMode` setting | **Done.** `Option` on `CreativeOpportunitiesConfig`. | +| 2 — head-seam neutrality gate | **Done.** `template_ad_slots_script`, three byte-identity tests. | +| 2b — body-close decoupling | **Done.** `BodyCloseInjection`, `body_close_injection`. | +| 2c — emit the marker under `Esi` | **Not done.** Blocked on the fragment endpoint; see below. | +| 3 — C2 eligibility gate | **Done.** `c2_bypass_reason`, eight tests. Logs only, no cache I/O. | +| 4 — C2 cache read/write | **Not started.** Design choice open; see below. | + +### What is deliberately absent + +**No marker is emitted under `Esi`.** The marker must point at a fragment endpoint +returning an **executable script**. `/_ts/page-bids` returns JSON +(`publisher.rs`, `handle_page_bids`) and ESI splices fragment bytes verbatim, so aiming +at it would put raw JSON where a script belongs. That endpoint does not exist, and a +marker with nothing behind it is worse than no marker. A test pins the current answer so +it changes deliberately. + +**No cache read or write.** `c2_bypass_reason` has a real call site that logs its verdict, +which makes the decision observable during the spike without mutating anything. Task 3 +Step 4 is blocked on choosing between read-through-with-body-transform and explicit +`cache::core` — the plan names that as a decision to make before writing code, and it is +under investigation rather than assumed. + +### A defect this work introduced and then caught + +Gating the head seam on neutrality made `ad_slots_script` `None` under the shared modes. +The body-close element handler read exactly that value to decide whether to inject at all, +so shared modes silently stopped injecting anything at `` — a side effect of a +`` change. Safe, since emitting nothing cannot leak, but wrong in the way the spec +warns about: the gate has to be "did this response carry bids", not "does this page have +slots". + +Found by reading the handler while starting the next step, not by a failing test. Fixed by +replacing the inference with a named decision. The test that now guards it asserts +body-close is identical whether or not the head script is present — a decision that read +the head script would be _accidentally_ correct today, because that script is always +absent under shared modes, and wrong the moment that changes. + +Worth recording because it is the same shape as the bug the whole task exists to prevent: +something that looks correct and quietly does nothing. + +### Coverage and its limits + +Fourteen new tests. `fmt`, all six clippy targets, and all four adapter suites pass, with +1850 core tests under Viceroy. `clippy --all-targets` caught a benchmark construction site +that all four test suites missed — the suites are not the whole gate. + +**The neutrality guarantee is narrower than it looks.** The tests prove `tsjs.adSlots` is +neutral. They say nothing about the other things injected at the same seam — integration +`head_inserts`, the gpt-diagnostics bootstrap, the RSC placeholder rewriter — which the +spec flags as needing an audit and which that audit has not yet covered. Until it does, +treat request-neutrality as asserted for one element rather than established for the +template. + +## Code review of the Task 3 commits — three HIGH findings, all closed + +**Date:** 2026-08-11. An independent review of the four implementation commits found +three HIGH issues. The default `Inline` path was verified unchanged byte-for-byte, so +none was a live regression — but all three were invariants this branch exists to +establish and none was enforced or tested. + +### 1. The auction dispatched under shared modes with nothing to consume it + +`assembly_mode` was computed _after_ the dispatch decision, so flipping to `client_fill` +or `esi` would still have sent real SSP bid requests, held the response for the full +auction budget, and discarded the result — because both injection seams now return +nothing — with no error, no warning and no log. + +Exactly the silent-waste signature §5 of the design doc is about, reached by an +incomplete feature flag rather than by removing the hold. Fixed by hoisting +`assembly_mode` above the dispatch and gating on `root_auction_is_useful`. + +The test derives the invariant rather than asserting per-variant: a root auction is +useful exactly when a seam will consume its result. A new mode cannot make the dispatch +gate and the injection decisions disagree without failing it. + +### 2. The C2 gate ignored the forwarded client `Cookie` + +TS forwards client cookies to origin unchanged — there is no `Cookie` strip on the +publisher path. So a response can be cookie-personalized while carrying no `Set-Cookie` +itself (session established earlier), no `Cache-Control` at all, status 200, HTML — and +every condition in the gate reported it cacheable. + +§4 of the design doc names this. The plan's own Task 3 Step 3 checklist missed it, so +the implementation matching the checklist exactly still had the hole. Now disqualifying +until an origin `Vary` covering `Cookie` is verified. + +### 3. Request-neutrality was asserted for one element, not the seam + +The head seam still injected integration `head_inserts` and the GPT-diagnostics +bootstrap unconditionally. + +Audited both. **`head_inserts` is clean** — all three implementations (datadome, didomi, +gpt) take the context parameter unused, so output depends on configuration, not the +request. **GPT diagnostics is not** — cookie- or query-activated, and documented as an +immutable request-scoped decision. + +It did not leak, but only by coincidence: `requires_private_no_store()` is a strict +superset of the conditions under which either script is emitted, and that stamp lands +before the C2 gate reads response headers, so the gate refused. Two independent +conditions that happened to align, with nothing enforcing the relationship. + +Fixed on both sides — the processor receives no diagnostics decision under shared modes, +**and** a test enumerates every combination of the decision's three fields asserting that +anything which injects also requires the stamp. The gate is the guarantee; the invariant +test is the backstop if the gate is ever removed. + +### What this says about the tests that existed + +All three findings were in code the existing tests covered — and passed. The tests +exercised the pure decision functions with hand-built inputs and never the rendered +``/`` bytes. That is still true: **no test renders a full document through +`create_html_processor` and compares two requests byte-for-byte.** The plan's Task 3 +Step 2 requires exactly that, and it remains the most valuable missing test. + +### Reviewer's gate, adopted + +Do not proceed to Task 3 Step 4 (actual C2 read/write) or expose `AssemblyMode` to any +test or staging traffic until the full-document byte-identity test exists. The three +fixes above close the known holes; that test is what would catch the next one. + +## Task 3 complete — the C2 cache engages end to end + +`2db10639` (store), `2a2e6c6a` (lookup), plus `b688d667`/`577eb85a` for the `Vary` +handling. A second request for the same URL is now served without touching the origin, +byte-identical to what was stored. + +**Three problems only appeared once the code had to run**, none of them visible in the +plan or in review: + +1. **The key needed the origin's `Vary`, but a lookup precedes the fetch.** Resolved with + an operator-stated list plus a post-response drift guard that refuses to store under a + key that missed something. Spike-grade: a two-phase lookup is the correct answer and + doubles the lookups. +2. **The key carried the encoding the _origin_ chose**, which also does not exist at + lookup time — storing under `br`, looking up under `gzip, br`, a cache that never hits. + Now keyed on what was sent to the origin. +3. **Storing needs every transformed byte; streaming does not collect them.** Shared modes + take the buffered finalizer, branching on the store authorization rather than the + assembly mode, so `Inline` cannot reach it. + +Each was a case where the design read as complete and the implementation had a hole in +it. That is the same pattern as the three review findings above, arriving one layer down. + +**Verified by mutation, not just by green tests.** Disabling the lookup fails the hit +test, so the hit is the cache answering rather than the fixture answering twice; dropping +the `Authorization` re-check fails the authenticated test; reading only the first `Vary` +header value, and disabling the drift guard, each fail their own tests. The reviewer's +gate above was satisfied first: the byte-identity tests it demanded exist and were +themselves mutation-checked. + +**Still not deployable.** `ClientFill` and `Esi` render a template with a hole and +nothing filling it — Task 4 and Task 5. A cache that works is necessary, not sufficient. + +## Local end-to-end run — the Esi arm renders + +`viceroy serve` against a stub origin, config pushed into a scratchpad `fastly.toml` so +nothing tracked was modified. Served document: + +```html +

Stub article

+
+

Body copy.

+ +``` + +No executable ESI tag. One origin fetch for two requests. `private, no-store` on the hit. +Cached template 353 bytes against 467 served, so the cache holds the pre-assembly +template. All three fragment formats behave: script, JSON, and `400` on a typo. `Inline` +unaffected — two fetches for two requests, no C2 activity, no markers. + +### The bug only a running server could find + +With the auction **enabled**, C2 never engaged: two origin fetches, marker unresolved. + +TS stamps its own `private, no-store` when `should_run_ad_stack` is true. The C2 gate ran +after that stamp, read it as the origin's declaration, concluded `OriginNotShareable`, and +refused — **on every page that serves ads**, which is every page that matters. + +The more important half is why no test caught it. The fixture left the auction disabled +and passed `slots: &[]`, so `should_run_ad_stack` was false in every test, the stamp never +fired, and the ordering was unobservable. Every C2 assertion had been made against the one +configuration where C2's hardest condition does not apply. + +Demonstrated both ways: with the old fixture, reintroducing the bug passes all seven +tests; with the corrected fixture it fails six. + +### Pattern across this branch + +Five bugs now share one shape — compiled, passed every existing test, and were wrong: + +1. The head-seam gate silently disabled body-close injection (`d9e05973`). +2. The key held the encoding the origin _chose_, so the cache could never hit (`2a2e6c6a`). +3. A C2 hit served with no `Cache-Control` at all (`0adb578e`). +4. A C2 hit dropped its in-flight auction, billing SSPs for nothing (`b3ac59a6`). +5. The gate read TS's own header as the origin's (`4c557347`). + +Three were found by writing the test the plan asked for. One needed a running server. None +were found by review — including my own, twice over on the same gate. + +The stale-cache test is the same failure in miniature: it passed while never reaching +`is_stale()`, and only mutation testing exposed that. A test that passes for the wrong +reason is worse than no test, because it is counted as coverage. + +## Independent review — two blockers, and two reasons the cache would have measured nothing + +An independent reviewer read `main...HEAD` and, importantly, **demonstrated** findings by +running code rather than inferring them. Four things it found that review-by-reading had +not. + +**A POST was answered from a cached GET.** `handle_publisher_request` is the `*`-method +fallback route, so a publisher path that renders on GET and accepts a form or webhook on +POST reaches it for both. The origin never saw the mutating request; the caller got `200` +and a page. Fixed at key construction, since the key governs lookup and store alike. + +**`Vary: Accept-Encoding` disqualified everything.** The key has a dedicated +`accept_encoding` field, so such an origin is already keyed correctly — but the coverage +check consulted only the operator-configured list and reported a gap. Every compressing +origin sends that header, so **C2 would have stored nothing against any real origin**. +This is worse than a plain bug: the spike would have measured a hit rate near zero and +reported it as a result. + +**Cookies excluded essentially every repeat visitor.** Any cookie disqualified in both +directions, and TS sets its own identity cookie. The population that could ever see a warm +hit was roughly first-ever page views and cookie-less clients. The design notes called +this the "first-nav exception"; it is the common case, not the exception. Now opt-in via +`origin_is_cookie_independent`, with the `Vary: Cookie` drift guard overriding a wrong +assertion. + +**`ClientFill` had no end-to-end coverage.** The reviewer reintroduced a diagnostics leak +scoped to that mode and all 1889 tests passed. Investigation showed that specific mutation +is unreachable — `requires_private_no_store()` is a strict superset of the injection +condition and stamps before the gate reads headers — but only by a coincidence between two +independent conditions. Both the coverage gap and the coincidence are now pinned by tests. + +### What the review says about the review process + +The reviewer's confirmed findings all came from **running** something. Its clean bills — +no leak in the template itself, no `Inline` regression — came with positive evidence: +tracing that integration context types carry no per-reader field at all, and separately +proving the leakage test has teeth by breaking the store/assemble order and watching it +fail. + +Two of my own comments were wrong, and it caught both by checking rather than reading: +one claimed three call sites where there are two, the other described a dispatcher +mechanism that stopped existing when assembly moved to `CompletedRequest`. + +Verified afterwards against a running server with the origin advertising +`Vary: Accept-Encoding`: a cookie-bearing repeat visitor now costs one origin fetch across +two requests, and a POST still reaches the origin. + +## Step B — consumers of TS's own response headers + +Not yet run. + +## Step C — hold and origin fetch timings + +Not yet run. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md new file mode 100644 index 000000000..5b9a455e7 --- /dev/null +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -0,0 +1,967 @@ +# #1009 ESI Validation Spike + +> **HISTORICAL SPIKE — DO NOT IMPLEMENT.** This document records the investigation, +> including executable ESI tags, parser/subrequests, and a client-fill arm that were all +> removed. Every unchecked item below is historical, not remaining work. The accepted +> implementation keeps the public `esi` spelling but uses Fastly C2 plus exact byte-seam +> assembly. See +> [the merge-hardening design](../specs/2026-08-12-1009-esi-merge-hardening-design.md) and +> [implementation plan](./2026-08-12-1009-esi-merge-hardening.md). + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps +> use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Decide #1009 on evidence. Build a shared-template pipeline behind a flag, run +ESI and client-fill against it, and produce a decision record that either adopts ESI, +adopts client-fill, or rejects both — with the Fastly-only maintenance cost priced in. + +**Architecture:** +`origin → lol_html transform → fastly::cache::core → finalize headers → stream assembly`. + +Headers finalize **before** assembly, not after — streaming responses on this adapter +commit headers first and then pipe chunks, so nothing can be set once assembly starts. + +The transform emits **one unconditional marker at the body-close seam**. Not two: the +head seam is not a template hole, because `tsjs.adSlots` presence is request-gated +(Task 3 Step 2). The cached object is a shared template with no per-user bytes and no +request-dependent decisions. Assembly is either the `esi` crate (edge) or a client fetch +(browser), selected per request by the arm allocator so both are measured on one build. + +**Tech Stack:** Rust 2024, `wasm32-wasip1`, `fastly` 0.12.1 (`cache::core`, `http::purge`), +`esi` 0.7, `lol_html`, a real Fastly test service for cache behaviour. + +**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` — +read the 2026-08-10 correction at the top and +[§6.6](../specs/2026-08-08-esi-cacheable-root-validation-design.md#66-the-esi-pipeline-corrected) +before writing any code. + +**Control:** [the Stage 0 plan](./2026-08-08-1009-measurement-and-stage-0.md). Its +instrumentation and its bypass flag are prerequisites — this plan compares against them +and does not duplicate them. + +--- + +## Why this plan exists + +An earlier revision of the spec concluded ESI was structurally impossible. It was wrong: +`fastly::cache::core` provides the cache boundary natively, and purge runs inside Compute. +That correction reopens #1009 as an empirical question, and this plan is how it gets +answered. + +**What is genuinely uncertain**, and what each arm is for: + +1. Does a shared template plus per-request assembly beat today's inline path enough to + matter? +2. Does **edge** assembly (ESI) beat **client** assembly (a fetch) by enough to justify a + Fastly-only rendering path that must be maintained alongside the portable one? +3. Can per-user leakage be excluded across cold MISS, warm HIT, stale revalidation, + transform failure, and fragment failure? + +Question 3 is a gate, not a metric. A win on 1 and 2 with a failure on 3 is a rejection. + +## Three caches, never conflated + +The original error came from treating these as one thing. Every task below names which it +means. + +| # | Cache | Contents | Status | +| --- | --------------------------------- | ----------------------------- | ----------------------------------- | +| C1 | Origin read-through | raw origin bytes | Exists. Stage 0 turns it back on. | +| C2 | Shared transformed template | post-`lol_html`, pre-assembly | **New.** What this plan builds. | +| C3 | Assembled-response delivery cache | final per-user output | **Must never exist.** Not proposed. | + +If a task appears to require C3, stop — that is the leakage failure mode, not a design +option. + +## Arms + +Five, but only four are treatable as equivalent. + +| Arm | Root | Bids | Notes | +| ------- | ----------------------- | ---------------- | ---------------------------------------------------------- | +| **A0** | inline, C1 bypassed | inline `` | Today. The baseline. | +| **A1** | inline, C1 on | inline `` | Stage 0. Isolates the bypass from the template change. | +| **A2** | shared template from C2 | client fetch | Portable. Works on all four adapters. | +| **A3** | shared template from C2 | ESI at the edge | Fastly-only. The thing #1009 proposed. | +| **REF** | origin direct, TS off | publisher's own | **Reference, not an arm.** Different work, not comparable. | + +A0→A1 measures the bypass. A1→A2 measures the template split. A2→A3 measures edge versus +client assembly — **that difference is the entire case for ESI**, and it is the number +this plan exists to produce. + +**Do not compare A2 and A3 on root TTFB.** They serve the same C2 template, so their root +timings should be near-identical by construction; a null result there proves nothing. +ESI's claimed advantage is that bids arrive without a client round-trip, so measure: +**bids-ready time**, **`adInit` fire time**, and **first TS-attributed creative paint**. +Root TTFB stays as a guard that the template path did not regress, not as the comparison. + +REF is included because #1009 anchors on it, and excluded from pass/fail because TS-off +does no auction and no injection. Comparing against it measures the feature's existence, +not its implementation. + +--- + +## Task order and dependencies + +``` +Task 1 (esi compiles) ── DONE, PASS ──┐ + ├──> Task 3 (C2 cache) ─┬──> Task 4 (A2 client-fill) +Stage 0 plan (flag + instrumentation) ┘ ├──> Task 5 (A3 ESI) + └──> Task 6 (safety gates) + │ + Task 2 (real service) ─────────────────────────────┴──> Task 7 (decision) +``` + +**Task 2 is not a blocker on Tasks 3–6.** Everything those tasks need is exercisable under +Viceroy 0.17 — verified, see Task 2. The real service is required only for the +measurements Task 7 decides on, so provision it once there is something worth measuring. + +Task 6 runs against every arm, not once at the end. + +--- + +## Task 1: Confirm `esi` 0.7 builds on this toolchain + +Cheapest possible falsification. Do this before anything else. + +**Files:** `crates/trusted-server-adapter-fastly/Cargo.toml` + +- [ ] **Step 1: Add the dependency** + +```bash +cargo add esi@0.7 --package trusted-server-adapter-fastly +``` + +It belongs in the **Fastly adapter**, never in `trusted-server-core` — the crate is +hard-bound to `fastly::{Request, Response, Backend}` and core must stay portable. + +- [ ] **Step 2: Check it compiles for the real target** + +```bash +cargo check-fastly +``` + +Expected: clean. The crate declares edition 2021 with no `rust-version`, and pulls recent +`rand` and `nom`, so this is a genuine question on Rust 1.95.0 / `wasm32-wasip1`. + +- [ ] **Step 3: Check no shared dependency was forced to move** + +```bash +git diff --stat Cargo.lock +cargo check --manifest-path crates/trusted-server-integration-tests/Cargo.toml --tests \ + --target "$(rustc -vV | sed -n 's/^host: //p')" +``` + +**Correction, verified 2026-08-10:** an earlier revision of this step warned about a +desync between the root `Cargo.lock` and `crates/trusted-server-integration-tests/Cargo.lock`. +**That second lockfile does not exist** — the crate is a workspace member (root +`Cargo.toml:10`) and shares the root lockfile. The hazard cannot arise in that form. + +What does matter is whether adding `esi` forces an **existing** shared dependency to a new +version, since `regex`, `bytes`, and `log` are used across the workspace. Adding a new +major that coexists is harmless; moving an existing one is not. If one moves, fix with a +targeted `cargo update -p --precise ` — **never a full update**. + +**Already run and recorded** in [the findings](./2026-08-08-1009-measurement-findings.md): +no existing shared dependency moved. + +- [ ] **Step 4: Record and commit, or stop** + +**Task 1 is complete — verdict PASS, recorded 2026-08-10.** `esi` 0.7.1 compiles clean on +Rust 1.95.0 / `wasm32-wasip1`, all six clippy targets pass, and no existing shared +dependency moved. See [the findings](./2026-08-08-1009-measurement-findings.md). + +Had Step 2 failed, this plan would have stopped here with #1009 answered "not on this +toolchain." It did not. + +```bash +git add crates/trusted-server-adapter-fastly/Cargo.toml Cargo.lock +git commit -m "Add the esi crate to the Fastly adapter for the #1009 validation spike" +``` + +--- + +## Task 2: Local validation first, real service only for what needs it + +**Verified 2026-08-10 under Viceroy 0.17: the entire Core Cache surface this spike uses +works locally.** A probe exercised `cache::core::insert`, `lookup`, `finish`, `to_stream`, +and — the shape Task 3 Step 4 actually specifies — `Transaction::lookup`, +`must_insert_or_update`, `insert(...).surrogate_keys(...).execute_and_stream_back()`, and +hit-after-insert semantics. All passed. Recorded in +[the findings](./2026-08-08-1009-measurement-findings.md). + +That reorders this plan. An earlier revision made provisioning a Fastly service Task 2 and +a blocker on everything after it. It is not a blocker: **almost all of the correctness and +safety work is local**, and only the numbers and the cache topology need real +infrastructure. + +| Work | Where | +| ------------------------------------------------------------ | ------------ | +| C2 insert / lookup / transaction logic (Task 3) | **Local** | +| The `lol_html` transform and template byte-identity (Task 3) | **Local** | +| ESI assembly — the crate is pure Rust over `BufRead`/`Write` | **Local** | +| DCA off, dispatcher allowlist, injection refusal (Task 5) | **Local** | +| Fragment-failure degradation (Task 5) | **Local** | +| Header-finalization ordering, no-C3 assertions (Task 6) | **Local** | +| Cross-user leakage / request-neutrality gates (Task 6) | **Local** | +| Shielding behaviour | Real service | +| POP-level cache tiering (`x-cache`, `hit-state`, `age`) | Real service | +| Request collapsing under genuine concurrency | Real service | +| Stale revalidation timing at the edge | Real service | +| **Every performance number in Task 7's decision rule** | Real service | + +**So: build and prove correctness locally through Tasks 3, 5, and 6 before provisioning +anything.** If the design is wrong or leaks, that surfaces locally for free, and the +service is only needed once there is something worth measuring. + +Two caveats on the local scope. Viceroy is a single instance, so a passing `Transaction` +test proves the API works, **not** that collapsing behaves correctly under load. And local +timings are meaningless for the decision — do not let a fast local run substitute for +Task 7 evidence. + +### When the real service is needed + +- [ ] **Step 1: Provision it — after local correctness passes, not before** + +Separate from production. Confirm and record: whether the publisher backend is +**shielded**, and whether any Delivery service fronts the Compute service. Both change +what the numbers mean. + +```bash +fastly service list +fastly backend list --service-id --version latest +``` + +The shielding answer also settles an open question from the Stage 0 findings: #1009's +off-TS win came from a shield HIT, so whether the test service has one determines whether +its numbers transfer to production at all. + +- [ ] **Step 2: Extend the harness for lineage, not just correlation** + +The existing tester-cookie A/B has no way to join server timings to browser timings. A +root-only request ID is not enough either: under A3 the auction happens in a **fragment +subrequest**, so a root ID never reaches the auction telemetry. + +Propagate a **lineage ID plus the experiment arm** through the whole chain: + +``` +root request → C2 lookup → fragment subrequest → auction telemetry → browser render event +``` + +Generated at TS entry, forwarded into the fragment request, attached to the +`auction_events_raw` row, echoed as `x-ts-request-id`, and exposed to the browser harness +so render events carry it. Every timing log line includes both fields. + +Without this the experiment cannot join hold time, origin time, auction telemetry, browser +TTFB, and render outcome for the same pageview. **That is the difference between an +experiment and a pile of numbers.** + +- [ ] **Step 3: Capture C1 and C2 status separately** + +`x-cache`, `hit-state`, and `age` describe the **HTTP read-through cache (C1)**. They say +nothing about the **transformed-template cache (C2)**, which is a `cache::core` object +with no HTTP semantics. Recording only the former and calling it "cache status" would +attribute C2 hits and misses to the wrong tier. + +Emit both: the C1 headers as-is, plus an explicit `x-ts-c2` field carrying HIT / MISS / +STALE / BYPASS from the transaction outcome. Record the serving POP alongside. A median +that mixes cold-MISS and warm-HIT requests is meaningless, and arms cannot be compared +unless the mix is known — per tier. + +- [ ] **Step 4: Build a request-scoped arm allocator** + +`AssemblyMode` as specified in Task 3 is a **global** setting, but the sample plan below +requires randomized, non-sequential allocation. A global flip gives sequential blocks +instead, which confounds arm with time of day, cache warmth, and traffic mix. + +Allocate per request: hash the lineage ID into buckets, or key off the tester cookie. +The global setting stays as the kill switch and as the way to force a single arm; the +allocator is what the experiment actually uses. Record the assigned arm on every log line +and every telemetry row. + +- [ ] **Step 5: Define the sample plan before collecting anything** + +Write all of this into the findings document **before** the first measurement, and treat +it as fixed: + +| Element | What to state | +| -------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Allocation | Requests per arm per route, and how arms are assigned | +| Randomization | Randomized or blocked by route and cache state — not sequential runs | +| Pilot variance | A small pilot to estimate variance, before sizing the real run | +| MDE and power | The smallest difference worth detecting, and the N that detects it | +| CI method | Which interval, computed how | +| Warmup and carryover | How cold MISS is forced, how warm HIT is confirmed, and how one arm's cache state is prevented from contaminating the next | + +Rationale: this whole effort exists because #1009 drew a causal conclusion from N=4 that +did not survive contact with the code. Repeating that with more arms and no power +calculation would be worse, not better — it would look rigorous while being equally +unfalsifiable. + +--- + +## Task 3: Build C2 — the shared transformed-template cache + +The core of the spike. Behind a flag, default off. + +**Files:** + +- `crates/trusted-server-core/src/publisher.rs` — emit **one** unconditional marker at the body-close seam (see Step 2; the head seam is not a template hole) +- `crates/trusted-server-core/src/settings.rs` — the mode flag +- `crates/trusted-server-adapter-fastly/src/` — the `cache::core` read/write + +- [ ] **Step 1: Add the assembly-mode setting** + +```rust +/// How per-user ad state reaches the page. +/// +/// `Inline` is today's behaviour: bids injected before ``, root uncacheable. +/// `ClientFill` and `Esi` both serve a shared template from the transformed-template +/// cache and fill the holes afterwards. Spike-only — remove with the spike. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AssemblyMode { + #[default] + Inline, + ClientFill, + Esi, +} +``` + +Default `Inline` so the flag is a no-op until set. Note the hazards the Stage 0 plan +already documents: `Settings` carries `#[serde(deny_unknown_fields)]`, `ts config push` is +typed, and `Publisher` has a hand-written `Default` plus eight exhaustive test literals +and a live doctest. + +- [ ] **Step 2: Make the template strictly request-neutral** + +**The obvious design is wrong and would leak.** An earlier draft kept `tsjs.adSlots` in +the shared template on the grounds that it is per-URL. Its _content_ is per-URL; its +_presence_ is not. It is gated on `should_run_ad_stack` (`publisher.rs:2920-2927`), which +is `is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the first request to fill C2 would freeze **its own** consent decision, bot +classification, prefetch status, and kill-switch state into an object every later visitor +reads. A consent-denied first fill serves a no-ads template to consenting users; a +consenting first fill serves ad markup to a user who refused. + +**Rule: the template contains an unconditional inert placeholder and nothing else.** + +| Element | Where it lives | +| ------------------------- | -------------------------------------------------- | +| tsjs bundle script tag | Template — content-hashed, genuinely per-URL | +| URL rewrites | Template — per-host, in the cache key | +| `tsjs.adSlots` | **Fragment** — its presence is request-dependent | +| `tsjs.bids` | **Fragment** | +| GPT diagnostics bootstrap | **Fragment** — gated on a per-request cookie/query | + +Emit **one** unconditional marker at the body-close seam, identical on every request that +reaches the transform. Under `Esi` it is an executable ESI include tag; under +`ClientFill` it is nothing at all, with the client fetching unprompted. + +- [ ] **Step 3: Bypass C2 for anything that must not be shared** + +`cache::core` is not an HTTP cache — it will happily store whatever you hand it. Nothing +rejects private or authenticated responses for you. Refuse to insert when **any** holds: + +- The origin response carries `Set-Cookie`. +- The origin response is `private`, `no-store`, or `no-cache`. +- The request carried `Authorization`. +- The response is not 200 with an HTML content type. +- DataDome's request filter replaced the document. + +Audit every request-dependent rewrite before declaring the template neutral — the +integration head-inserts and the GPT-diagnostics bootstrap are both request-scoped and +must not reach C2. + +**Assert it, do not assume it.** A unit test over the transform output must fail on any +of: a bid value, an EC ID, a consent string, a geo value, a diagnostics bootstrap, or a +`Set-Cookie`. Then a second test must assert the template is **byte-identical** for two +requests differing in consent, bot classification, and prefetch status. That second test +is the one that catches this class of bug; the first would have passed on the broken +design. + +- [ ] **Step 4: Write and read C2 — with the real API** + +The builder is move-based and the insert and read handles are different objects. Naïve +code does not compile: + +```rust +// WRONG — surrogate_keys consumes the builder and returns it; this discards the +// return value and then uses a moved binding. And execute() gives a WRITE stream, +// so there is nothing to read back from it. +let mut insert = cache::core::insert(key, ttl); +insert.surrogate_keys(["ts-template"]); +let body = insert.execute()?; +``` + +Correct shape, using a transaction so a cold cache under load transforms once: + +```rust +use fastly::cache::core::{Transaction, CacheKey}; + +let tx = Transaction::lookup(CacheKey::from(key_bytes)).execute()?; + +// Order matters: a STALE entry sets BOTH found() and must_insert_or_update(). +// Testing found() first would serve the stale bytes and silently never fulfil the +// update obligation, leaving every concurrent waiter blocked until timeout. +let template: Body = if tx.must_insert_or_update() { + // Fetch and prepare BEFORE consuming `tx`. After `insert()` the transaction is + // gone and `cancel_insert_or_update()` is unreachable, so anything that can fail + // and does not need the writer belongs here. + let origin = match fetch_and_prepare_origin() { + Ok(origin) => origin, + Err(e) => { + tx.cancel_insert_or_update()?; // releases the obligation to a waiter + return fallback_uncached(e); + } + }; + + // `Transaction::insert(self)` consumes `tx` from this line on. + let (mut writer, found) = tx + .insert(template_ttl) + .surrogate_keys(["ts-template", &url_surrogate_key]) // chained, not discarded + .user_metadata(metadata_envelope) + .execute_and_stream_back()?; + + match stream_lol_html_output(origin, &mut writer) { + Ok(()) => { + writer.finish()?; // REQUIRED, and consumes `writer` + found.to_stream()? // fallible; there is no `to_body()` + } + Err(e) => { + // Also consumes `writer`, marking an unsuccessful end so no partial + // template is served. (A `StreamingBody` dropped without `finish()` is + // aborted anyway, but say it explicitly.) + writer.abandon()?; + return fallback_uncached(e); + } + } +} else if let Some(found) = tx.found() { + found.to_stream()? // C2 HIT — skip origin fetch and transform +} else { + unreachable!("a transaction is either obliged to insert or has found an item") +}; +``` + +Two ownership rules this shape exists to respect, both of which an earlier draft broke: +`Transaction::insert(self)` **consumes** the transaction, so a helper taking `&tx` cannot +call it and `cancel_insert_or_update` is unreachable afterwards; and `finish`/`abandon` +each consume the writer, so neither can be referenced from an arm that did not bind it. + +**Decide the stale policy explicitly.** `Found::is_stale()` and `is_usable()` exist, and +`stale_while_revalidate` can be set at insert. Serving stale while revalidating is a real +option — but it is a state machine, and `cache::core` implements none of it for you. The +spike should start by treating stale as a miss and only add stale-serve if the numbers +justify it. + +**`cache::core` carries no HTTP semantics.** Status, headers, content encoding, and +revalidation are all yours. Serialize what you need into `user_metadata` — at minimum the +content encoding, the transform schema version, and the origin `Vary` values the key was +built from — and decide explicitly whether the stored template is compressed. + +**Cache key must include**, beyond the origin's declared `Vary` (`rsc`, +`next-router-state-tree`, `next-router-prefetch`, `next-router-segment-prefetch`, +`Accept-Encoding` — measured, see the Stage 0 findings): + +- The full URL, explicitly. Do not rely on an ambient request key. +- **The assembly mode.** A2 and A3 emit different template bytes and would otherwise + poison each other's entries. +- **A template schema version**, bumped whenever the transform changes, so a deploy does + not read yesterday's shape. +- Request host and scheme, the enabled-integration set, and the tsjs content hash. + +Per-user signals must never appear in the key. If a signal cannot be excluded from the +template, it does not belong in C2 at all. + +### Design decided 2026-08-10: `cache::core`. Do not revisit read-through. + +An earlier revision left this open between `cache::core` and read-through caching with +`after_send` + `set_body_transform`. Investigated and verified against the pinned SDK and +Viceroy 0.17 source. **Read-through is not viable here** — not on preference, on three +hard blockers: + +1. **Viceroy stubs the entire HTTP Cache ABI**, and the SDK converts that into a _send + error_ rather than a fallback. `is_request_cacheable` returns + `Err(NotAvailable("HTTP Cache API primitives"))` + (`viceroy-lib-0.17.0/src/wiggle_abi/http_cache.rs:108-114`; 26 such stubs in that + file), which makes `must_use_host_caching()` true, which with a send hook set returns + `Err(SendErrorCause::HttpCacheApiUnsupported)` + (`fastly-0.12.1/src/http/request.rs:626-632`). **Setting `after_send` makes every + publisher origin fetch fail** under `fastly compute serve`, `cargo test-fastly`, and + the parity suite. The whole local loop dies. +2. **`with_cache_bypass` makes the hook silently dead.** `get_caching_mode` checks + `cache_override.is_pass()` **first** (`request.rs:612-615`) and returns host caching, so + `after_send` is never invoked and no error is raised. On exactly the requests in scope, + today, the hook would do nothing quietly. +3. **The closure bounds are incompatible with this codebase.** `with_after_send` requires + `Fn + Send + Sync + 'static` (`request.rs:545-550`). Everything the rewriter needs is + `!Send` by construction — `edgezero_core::body::Body` wraps a `LocalBoxStream` + deliberately, which is why the platform layer is `#[async_trait(?Send)]` throughout. + And `set_body_transform` is synchronous, so it could never await the auction collect. + +Read-through's appeal was real — `CandidateResponse::apply_and_stream_back` is +`execute_and_stream_back` with HTTP semantics attached, and TTL/SWR/vary/surrogate keys +derived from origin headers for free. It is simply unreachable from here. + +**Also settled: core cannot reach it at all.** `PlatformHttpRequest` +(`platform/http.rs:16-37`) is a plain data struct with no callback slot, and carrying one +would name `fastly::http::CandidateResponse` in portable core, breaking the other three +adapters. + +### Follow the existing null-object pattern + +`cache::core` fits the shape the repo already uses four times for a Fastly-only capability +behind a portable trait: `UnavailableHttpClient` (`platform/http.rs:216-243`), +`UnavailableKvStore` (`platform/kv.rs:14-17`), and the `RuntimeServices.kv_store` +field/accessor/builder (`platform/types.rs:170,222,269,330`). Add +`PlatformTemplateCache` the same way, and follow +`crates/trusted-server-adapter-fastly/src/ec_kv.rs` — 140 lines, the repo's only real +edge-storage read/write — rather than inventing a shape. + +**Return `EdgeBody`, not `Vec`.** `EdgeBody::Stream` exists, +`fastly_body_to_edge_stream` (`adapter-fastly/src/platform.rs:503`) already converts, and +`PublisherResponse::Buffered` tolerates a live stream (`publisher.rs:1019-1022`). + +### Exact insertion point + +**Immediately before `let mut platform_request = PlatformHttpRequest::new(...)`** — the +last line before `req` is consumed, and a few lines before the origin send. Everything +needed is in scope there: `settings`, `services`, the final URI and Host, `backend_name`, +`request_path`, `matched_slots`, `should_run_ad_stack`, `request_had_authorization`, +`request_host`, `request_scheme`. + +**One required move:** `assembly_mode` is currently computed _after_ the send, for the +logging call site. It depends only on `settings`, so hoist it above the insertion point. + +**Tee-ing is not needed.** With any post-processor registered — and the Next.js +integration always registers one — `HtmlWithPostProcessing` emits nothing until the final +chunk and then returns the whole transformed document as one contiguous buffer +(`html_processor.rs:92-97,148`). Two `write_all` calls on the same slice; no tee +abstraction, no extra copy. Still use `execute_and_stream_back`, but for transaction +correctness and request collapsing rather than for memory. On a hit the processor is never +built at all. + +- [ ] **Step 4b: close the risks the design investigation surfaced** + +Four, all specific to this codebase rather than to `cache::core` in general. + +**`Vary` is in the key list but nothing consumes it.** `c2_bypass_reason` checks +`Set-Cookie`, `Cache-Control`, `Authorization`, status and content type — **not `Vary`**. +Viceroy supports `WriteOptions.vary_rule`, so the mechanism exists; the gate has to use +it. Until then the key is missing a signal the origin explicitly declares, and Step A's +verdict is a `PROVISIONAL PASS`, not a release gate. + +**Resolved — `VarySpec`, commit `b688d667`.** Building the key exposed a problem this +plan states but does not solve: the key must cover everything the origin varies on, but +**a lookup happens before the fetch**, so on a cold key the origin's `Vary` is not yet +known. Three ways out — configure the list; two-phase lookup against a URL-keyed record +holding the last-seen `Vary`; or store the list alongside and re-key on mismatch. The +latter two are correct and double the lookups on every request. + +Configured is taken, **as a spike-grade choice rather than a production one**: Step A +already measured the origin's actual `Vary`, and a 60s TTL bounds drift to a minute +rather than indefinitely. + +The drift is guarded rather than merely accepted. `VarySpec::uncovered_by` runs _after_ +the origin responds, when its `Vary` is finally known, and names which headers the +configured spec missed. A template built under a key that did not cover something the +origin varies on **must not be stored** — a request differing only in that header would +read it. Naming the specific headers makes a stale config identifiable instead of +producing a generic refusal. + +Two decisions worth their tests. An absent header and a present-but-empty one key the +same, because the origin sees no difference between them. And `Vary: *` is not reported +as a named gap — it means uncacheable, which the eligibility gate handles, and reporting +it would produce a nonsense instruction to configure a header called `*`. + +Still open: wiring `uncovered_by` into `c2_bypass_reason` as a bypass reason, which +happens with the store call site. + +**Store bytes plus a metadata envelope; rebuild every header on a hit.** The publisher +path forces `private, no-store` and strips `ETag`/`Last-Modified`/CDN headers _after_ the +send. Replaying stored origin headers would fight that. Store only the transformed body +and a small `user_metadata` envelope — content encoding, content type, schema version, +tsjs hash — and construct every response header from scratch on a hit. Then no origin +header is ever replayed and the `Set-Cookie` privacy net is trivially safe. +`get_user_metadata` is implemented in Viceroy. + +**Content-Encoding belongs in the key.** The streaming pipeline pairs input encoding to +the same output encoding, so the transformed bytes inherit whatever the origin negotiated +from the client's `Accept-Encoding` — still gzip, deflate, br or identity after +`restrict_accept_encoding` narrows it. Either key on the negotiated encoding or normalize +to identity in the cache and re-encode on read. Getting this wrong serves brotli bytes to +a client that asked for gzip. + +**Host and scheme belong in the key.** The post-processed output is host-dependent by +construction: `request_host` and `request_scheme` reach `IntegrationHtmlContext`. + +- [ ] **Step 4c: file the wasted-dispatch follow-up** + +The auction is dispatched _before_ the insertion point. Under `Esi` and `ClientFill` the +root injects nothing, so that dispatch is already pure waste on this branch — and on a C2 +hit it is waste that must be cleaned up via `emit_abandoned_auction` or it leaks +telemetry. + +Keeping the lookup at the insertion point above is right for the spike: minimal diff, and +lookup latency overlaps the in-flight auction. Moving it earlier would eliminate the +wasted dispatch but serialize the lookup ahead of dispatch. **File it; do not fix it +here.** Suppressing root-level dispatch under the shared modes is Task 4's job, where it +also has to be reconciled with the exactly-one-auction gate. + +- [ ] **Step 5: Unit tests, then the target suite** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin assembly_mode +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +cargo fmt --all -- --check && cargo clippy-fastly +``` + +`ClientFill` must work on all four adapters. `Esi` is Fastly-only and must not break the +others' compilation. + +- [x] **Step 6: the call site — DONE.** `2db10639` (store), `2a2e6c6a` (lookup). + +The cache now engages end to end: a second request for the same URL is served without +touching the origin, and is byte-identical to what was stored. Verified by mutation — +disabling the lookup fails the hit test, so the hit is the cache answering rather than +the fixture answering twice. + +**Wiring the lookup corrected the key.** It carried the content encoding the _origin_ +chose, which does not exist at lookup time. That meant storing under `br` and looking up +under `gzip, br` — a cache that never hits. The field is now the `Accept-Encoding` sent +to the origin. Sound because negotiation is a function of what the origin was offered, +so identical offers yield identical choices; the chosen encoding stays in the metadata +and is what the served response declares. + +That made every key field request-derived, so **the key is built before the fetch** and +the response gate only authorizes storing it. A key that needed the response could only +ever authorize a store, never satisfy a read. + +**The lookup re-checks the request-derived disqualifications, and only those.** The +store gate is response-derived and cannot re-run, but need not: anything in the cache +passed it on the way in. What must re-run are properties of the _reader_ rather than of +the bytes — an authenticated request must not be served a shared template even when that +template is perfectly cacheable. + +**Shared modes take the buffered finalizer.** Storing needs every transformed byte and +streaming does not collect them. The branch keys on the store authorization rather than +on the assembly mode, so `Inline` never reaches it and the spike cannot regress the +shipped path by construction. A C2 _miss_ therefore buffers — the right trade, since a +miss is already paying an origin fetch and a full transform, and what the spike measures +is the hit, where there is no origin fetch to stream from at all. + +Every response header on a hit is constructed, never replayed, so no origin header can +reach a second visitor through the cache. + +The publisher tests use an in-memory cache double, so they prove the wiring rather than +the backing. The join they leave untested is the one `app.rs` makes: the publisher +reaches the cache as a `dyn PlatformTemplateCache` behind `RuntimeServices`, never as +the concrete type the Fastly tests exercise. That join is now executed under Viceroy +against the real Core Cache rather than only type-checked. + +**What this does not establish.** `ClientFill` and `Esi` still render a template with a +hole and nothing filling it. Task 4 and Task 5 remain the blockers on anything +deployable — a cache that works is necessary, not sufficient. + +--- + +## Task 4: Arm A2 — client-fill + +Mostly already specified. See +[the spec's Appendix B](../specs/2026-08-08-esi-cacheable-root-validation-design.md#appendix-b--stage-1-plumbing-condensed) +for the client plumbing, the two-condition join gate, and the server contract; and +[§5](../specs/2026-08-08-esi-cacheable-root-validation-design.md#5-the-trap-in-the-deferred-work--read-this-before-scheduling-stages-12) +for the silent-empty-bids trap, which applies in full. + +- [ ] **Step 1: Hoist the closure-trapped client state** — `pageBidsEndpoint`, + `requestPageBids`, and the `inflight`/`currentPath`/`lastAppliedPath` state, per + Appendix B. Do **not** route the initial load through `onNavigate`. +- [ ] **Step 2: Make `installScheduleInitialAdInit` a hydration-ready AND bids-settled + join**, with a bounded timeout that fires `adInit` untargeted rather than stranding + the slot. Derive the timeout from measured fetch latency, not a constant. +- [ ] **Step 3: Suppress the navigation-path dispatch** so exactly one auction runs per + pageview. Add a new `AuctionSource` for initial loads **plus the mechanism that + delivers it** — a header behind the same-origin gate, not a query parameter. +- [ ] **Step 4: Relocate terminal telemetry.** Navigation `Completed` is emitted only from + the collect functions; the `ts-debug` dump rides the same string. Both move. +- [ ] **Step 5: Verify exactly one auction per pageview** in `auction_events_raw`. Two is + a doubling of SSP spend and an immediate fail. + +--- + +## Task 5: Arm A3 — ESI at the edge + +- [x] **Step 0: the mechanism works — DONE.** `9539061e`, hardened in `0597f54e`. + +Verified under Viceroy with the real `esi` 0.7 crate rather than argued from docs: a +template carrying the `` seam's own ESI include tag comes back with the fragment +spliced in its place and no unresolved tag left. + +**The async/sync obstacle is dissolved, not worked around.** `esi`'s fragment dispatcher +is synchronous and this codebase's fragment producer is `async`; calling one from the +other means a nested executor, which panics. +`PendingFragmentContent::CompletedRequest` lets the dispatcher hand back an +already-built response, so the caller resolves the fragment in the normal async flow and +the dispatcher performs **no I/O at all** — no subrequest, no backend, no self-call, +nothing for Viceroy to stub. That also removes the need for a self-referencing backend +this plan would otherwise have required. + +**Step 2's instruction was right, and reading the crate showed why.** +`CacheConfig::is_includes_cacheable` defaults to **`true`**. A fragment carries one +visitor's bids, so the default caches per-user data and serves it to the next visitor — +silently, on a hit. `includes_force_ttl` is worse where set: it caches everything, +ignoring `private`, `no-store` and `Set-Cookie` alike. Both now stated explicitly, along +with `default_dca`/`inherit_parent_dca` (fragment bytes are data, never re-parsed as +ESI), `max_include_depth = 1`, and rendered caching / `edge_control` off because the +publisher path owns those headers. + +Nine tests. Four assert the configuration; the rest assert behaviour, including that a +fragment containing its own nested ESI include is spliced as text rather than dispatched, so +auction data cannot drive fragment requests. + +**What remains is the call site**, below. Emitting the include and resolving it are both +proven; connecting them is not done. + +- [ ] **Step 1: Wire `process_stream`, not the wrappers** + +`process_response` and `process_response_streaming` consume `self` _and_ send the response +themselves, which takes ownership away from the finalize / `ec_finalize` / apply-effects +ordering. `process_stream(&mut self, src: impl BufRead, out: &mut impl Write, …)` keeps it. + +Source is the C2 body. Sink is the client response body. + +**The ordering an earlier draft described is impossible.** It said EC cookie, geo, and the +privacy net run _after_ assembly. They cannot: streaming responses on this adapter +**commit headers first and then pipe chunks** +(`adapter-fastly/src/main.rs`, `send_edgezero_response`). Once ESI starts writing, no +header can change. + +The correct invariant: + +> **Finalize every header before a single body byte is written** — EC `Set-Cookie`, geo +> suppression, and an unconditional `Cache-Control: private, no-store` — **then** stream +> the assembly with no further header mutation. + +That means `private, no-store` is set unconditionally up front rather than derived from +what the assembly turns out to contain. Deriving it after the fact is not available, and +assuming it was is how a per-user response ends up shared-cacheable. + +- [ ] **Step 2: Disable DCA explicitly and allowlist the dispatcher** + +```rust +let config = esi::Configuration::default() + .with_escaped(false) + .with_default_dca(esi::DcaMode::None) // call the setter; do not rely on the default + .with_inherit_parent_dca(false); +``` + +Comments are not configuration. An earlier draft said DCA "stays at its default" — on a +pre-1.0 crate whose default could move in a patch release, and where this setting fails +**open**, that is not good enough. Call the setters. + +Also disable **fragment caching** explicitly, or mark the include `no-store="on"`. A +cached auction fragment is a per-user object in a shared cache — the C3 failure mode by +another route. + +The dispatcher must be **exact-path allowlisted**: a fragment URL that is not the bids +endpoint is refused, not fetched. The built-in dispatcher builds a dynamic backend per URL +host and panics on a hostless URL — never use it. + +Rationale in the spec's §2: bid payloads carry partner-controlled creative markup, so a +recursive parse would let an SSP make the edge fetch an arbitrary URL. **Add a unit test +that feeds a partner-controlled ESI include targeting `http://attacker.example/` through +a creative payload and asserts no fetch is attempted.** + +- [ ] **Step 3: The fragment must be a script, not the JSON endpoint** + +**`/_ts/page-bids` cannot be the ESI target.** It returns +`serde_json::json!({"slots":…, "bids":…})` (`publisher.rs:3987`), and ESI splices fragment +bytes in literally — the page would contain raw JSON where an executable script belongs. +Nothing would call `scheduleInitialAdInit`. + +Add a **dedicated fragment endpoint** returning the executable script — the same shape +`build_bids_script` produces today, plus the `adSlots` assignment that moved out of the +template in Task 3 Step 2. Either that, or use the `esi` crate's fragment-response +processor to wrap the JSON; the dedicated endpoint is simpler and easier to assert on. + +Three more things the naïve marker gets wrong: + +- **The same-origin gate will reject it.** `page_bids_request_allowed` + (`publisher.rs:3644`) requires `Sec-Fetch-Site: same-origin` or the `X-TSJS-Page-Bids` + header. An internal ESI subrequest carries neither. Give the fragment endpoint an + internal contract and a fixed backend rather than weakening that gate — it exists to + stop third parties burning SSP quota. +- **Parent context does not propagate.** EC identity, consent state, client IP, geo, User + Agent, and the correlation ID all live on the parent request. Forward an **explicitly + approved allowlist** of them into the fragment request. Forwarding everything is how a + fragment ends up more privileged than the parent. +- **Root dispatch must be suppressed.** The navigation path already dispatches an + auction. If A3 does not suppress it, every pageview runs two — doubling SSP and APS + spend. This applies to **A2 and A3 alike**. + +- [ ] **Step 4: Validate the whole URL, not the path** + +An exact-path allowlist alone permits `https://attacker.example/_ts/page-bids`. Validate +**scheme, authority, method, path, and query** — or better, ignore the marker's URL +entirely and dispatch to a fixed internal backend, treating the ESI include as a signal +rather than an address. + +Add a test that feeds an ESI include targeting +`https://attacker.example/_ts/page-bids` through a creative payload and asserts no +outbound fetch is attempted. + +- [ ] **Step 5: Deterministic synthetic fragment first** + +Before wiring the real auction, point the include at a fixed-content endpoint. This +separates "does the pipeline assemble correctly" from "does the auction behave," and the +two fail very differently. Only once assembly is proven does the fragment become the real +one. + +- [ ] **Step 6: Handle the flush hazard** + +`esi` flushes its output writer after each parse batch. Fastly's `StreamingBody` is a +`BufWriter`, so anything between esi and it must propagate `flush()` or nothing leaves the +Wasm heap. + +- [ ] **Step 7: Fragment failure must degrade, not break** + +Assert that a fragment timeout or non-2xx yields a page with empty bids rather than a 5xx +or a truncated document. Note the crate's non-obvious semantics: `alt` is attempted before +`onerror="continue"`, and `` runs **all** attempts and concatenates every +non-failed output — it is not first-success-wins. + +--- + +## Task 6: Safety gates — run against every arm + +Not a phase. Every one of these is a hard fail, independent of any performance result. + +- [x] **Zero cross-user leakage.** DONE — `76df2469`. Two synthetic users differing in EC + identity, consent jurisdiction and geo store a byte-identical template, each against + a fresh cache so the first cannot answer for the second. Forbidden-substring checks + are the second layer, since byte-identity also holds if both leak the same thing. + Mutation-verified: leaking `adSlots` through the head seam fails it. +- [x] **Cold MISS, warm HIT, stale revalidation** DONE — `76df2469`, and end to end under + `viceroy serve` (below). Stale reads as a miss; serving stale would mean serving a + template built by an older transform or bundle. + + The first stale test passed for the wrong reason and had to be rewritten: a zero TTL + produces an *absent* entry, not a stale one, so `is_stale()` was never reached — + confirmed by reverting the check and watching it stay green. Only a + `stale_while_revalidate` window makes an entry present-and-stale. + +- [x] **Transform failure** DONE — `76df2469`. A partial template in C2 is the worst + outcome available: a truncated document served to every later visitor, indefinitely, + with no error after the first request. Mutation-verified by storing before the cap + check. +- [ ] **Request collapsing** works: concurrent cold requests transform once. +- [x] **DCA disabled** DONE — `0597f54e`. Config asserted _and_ behaviour: a fragment + carrying its own nested ESI include is spliced as text rather than dispatched. + +- [ ] **Request collapsing** — not tested, and not testable here. Viceroy is + single-threaded, so the concurrent cold-request case cannot be produced. The racing + _writer_ path is covered (`a_second_put_on_a_fresh_entry_is_a_no_op`), which is the + correctness half; the collapsing half needs real concurrency. +- [ ] **Exactly one auction per pageview**, from `auction_events_raw`. +- [ ] **Cookie and privacy finalization ran BEFORE assembly**, not after — EC + `Set-Cookie` on first visit, geo suppression, and an unconditional + `Cache-Control: private, no-store`. Headers commit before the body streams on this + adapter, so "finalize after assembly" is not available; asserting it that way is how + a per-user response ends up shared-cacheable. ESI's streaming mode dropping + `$add_header` is a consequence of the same constraint, not a separate hazard. +- [ ] **Slot and bid attribution unchanged.** Same slots matched, same bids applied, same + renders attributed. Use TS-attributed renders — the SSAT line item, non-empty + `ts.bids`, `hb_adid` presence — **never slot fill**, which is blind to empty bids + because `adInit` defines slots regardless. +- [x] **No C3 — assert positively, not by absence.** DONE — `0adb578e`, and this gate's + wording caught a live bug. A C2 hit returns before the point where the publisher path + stamps `private, no-store`, so it served HTML with **no `Cache-Control` at all** — + heuristically cacheable, and therefore a shared cache of an assembled per-user + response. Checking for the _absence_ of `public`/`s-maxage`/`Surrogate-Control` would + have reported it as safe, because there was nothing present to forbid. Covered for + returning visitors specifically, where the cookie-privacy net never fires. + + Original wording, retained because it is what made the difference: Forbidding `public`, `s-maxage`, and + `Surrogate-Control` is **not sufficient**: a bare `Cache-Control: max-age=60` passes + that check and is still shared-cacheable, and that is exactly what the measured + origin sends. Require instead that every assembled response carries + `Cache-Control: private, no-store` and that `Expires`, `ETag`, `Last-Modified`, and + all four CDN cache directives are stripped. Test it for **returning** users + specifically — they set no EC cookie, so the cookie privacy net never fires and is + not a backstop here. + +--- + +## Task 7: The decision record + +**Files:** `docs/superpowers/plans/2026-08-10-1009-esi-decision-record.md` + +- [ ] **Step 1: Record every arm** with N, confidence interval, cache-tier mix, route mix, + and POP. Any arm missing those is not reportable. + +- [ ] **Step 2: Apply the decision rule, stated here before the data exists** + +**Adopt ESI only if all three hold:** + +1. Every Task 6 gate passes on A3. +2. A3 beats A2 on **bids-ready time, `adInit` fire time, and first TS-attributed creative + paint** — by a margin the reviewers ratify **before** collection, not chosen after + seeing the numbers. **Not root TTFB:** A2 and A3 serve the same C2 template, so their + root timings are near-identical by construction and a difference there would be noise. + Root TTFB is a non-regression guard only. +3. Render outcomes on A3 are non-inferior to A0. + +**Otherwise adopt A2 (client-fill)** if its gates pass and it beats A1. It is portable +across all four adapters and carries no Fastly-only maintenance burden. + +**Otherwise keep A1** — Stage 0 alone — and record #1009 as answered in the negative with +evidence. + +The margin in (2) exists because A3's cost is not its diff. It is a second rendering +architecture, Fastly-only, on a pre-1.0 crate, in the critical render path. A small win +does not pay for that. + +- [ ] **Step 3: Record what would change the answer**, so this does not get re-litigated + from scratch. At minimum: React #418 / [#938](https://github.com/IABTechLab/trusted-server/issues/938) + being fixed such that `adInit` can run synchronously, which is what would make edge + assembly's round-trip saving actually worth something. + +- [ ] **Step 4: Clean up.** Remove the spike flag or promote it to a real setting; purge + C2 (`purge_surrogate_key` on `ts-template`); remove the synthetic fragment endpoint; + and either land or delete the `esi` dependency. **A spike flag left in place becomes + permanent configuration surface.** + +--- + +## Reproducibility metadata + +Record with every result, or it cannot be re-run or trusted: commit SHA; `esi` and +`fastly` crate versions; Fastly service and version IDs; whether the backend is shielded; +`template_ttl`; the origin's `Cache-Control` and `Vary` at collection time; assembly mode; +routes; N per arm; and the cache-tier mix. + +## Out of scope + +- **Stages 1–2 of the spec** as production work. This spike may build parts of the + client-fill path to measure it; shipping it is a separate decision behind the + correctness defects. +- **Full RSC/flight partitioning.** `rsc_flight.rs` has no static/dynamic split. +- **Publisher-authored ESI.** Breaks the no-origin-changes promise. +- **A C3 delivery cache.** Not a deferred item — a thing that must not exist. + +## Definition of done + +- [ ] Task 1 verdict recorded: `esi` 0.7 builds on Rust 1.95.0 / `wasm32-wasip1`, or it + does not and the spike stopped. +- [ ] All four arms measured on one build, with correlation IDs joining server and browser + timings, and cache tier recorded per request. +- [ ] Every Task 6 gate has an explicit pass/fail per arm. +- [ ] Decision record exists, applies the pre-ratified rule, and names what would change + the answer. +- [ ] Cleanup complete: flag resolved, C2 purged, synthetic endpoint removed, dependency + landed or dropped. +- [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four + adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md new file mode 100644 index 000000000..a2b388594 --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md @@ -0,0 +1,294 @@ +# #1009 ESI Merge and Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:executing-plans` to implement +> this plan task-by-task. This plan is intentionally executed inline because the operator +> explicitly prohibited subagents. + +**Goal:** Merge current `main` and make the opt-in ESI byte-seam/shared-template path correct, +private, cache-semantic, compressed, observable, and operationally reversible. + +**Architecture:** Fastly Core Cache holds identity-encoded reader-neutral templates behind a +transaction acquired before origin work. Every request assembles its own slots and structured bid +map at an exact inert seam, encodes the result for that client, and receives a final immutable +private/no-store policy. + +**Tech Stack:** Rust 1.95, Fastly Compute/Core Cache, `edgezero_core` HTTP types, `lol_html`, +TypeScript/Vitest, Viceroy, shell harness. + +> **Implementation status, 2026-08-12:** Tasks 1–12 are complete on the branch. The Viceroy +> harness passed in both modes after running outside the filesystem sandbox so it could read the +> macOS native-certificate keychain. + +--- + +### Task 1: Merge live main and preserve auction contracts + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/integrations/gpt_diagnostics.rs` +- Modify: `crates/trusted-server-js/lib/src/core/types.ts` +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Test: adjacent Rust and Vitest modules + +- [x] Merge `origin/main` with `git merge --no-ff origin/main`. +- [x] Resolve the `AdBidsState`/`write_bids_to_state` conflict by building one structured map with + `auction_id`, storing both map and script, and returning its delivered slot IDs. +- [x] Add/adjust tests proving ESI and inline retain `hb_auction_id`, APS renderer metadata, and + delivered-winner attribution. +- [x] Run the focused Rust and GPT tests. +- [x] Complete the merge commit. + +### Task 2: Remove mechanisms outside the approved ESI byte-seam design + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Delete: `crates/trusted-server-core/src/platform/template_assembly.rs` +- Modify: `crates/trusted-server-core/src/platform/mod.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Delete: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-adapter-fastly/Cargo.toml` +- Modify: `Cargo.lock` + +- [x] Update mode tests to specify only `inline` and `esi`; watch the old client-fill expectations + fail or stop compiling. +- [x] Remove `ClientFill`, executable fragment serialization, assembler traits/registration, and + the `esi` crate. +- [x] Update comments to call the production path byte-seam assembly. +- [x] Run focused configuration, publisher, and Fastly adapter tests. +- [x] Commit the scope cleanup. + +### Task 3: Canonicalize and bound the template key + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write failing tests for absent versus empty `Vary`, repeated raw values, invalid configured + names, punctuation-colliding purge URLs, changed origin host override, and changed creative + configuration. +- [x] Replace string pairs with a typed canonical `Vary` value preserving presence and all bytes. +- [x] Hash a length-prefixed canonical key and hash the URL-specific surrogate key. +- [x] Include publisher origin identity and the complete template-shaping fingerprint. +- [x] Run focused key/configuration tests and commit. + +### Task 4: Enforce request and origin cache semantics + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/Cargo.toml` if HTTP-date parsing needs a direct dependency + +- [x] Write failing tests for response `max-age=0`, positive max age, repeated/malformed cache + directives, `Age` exhaustion, expired/malformed `Expires`, missing freshness, invalid `Vary`, + and request no-cache/no-store/range/conditional bypasses. +- [x] Add a typed cache eligibility result carrying the positive remaining TTL. +- [x] Parse relevant response directives fail-closed and cap, never extend, origin freshness. +- [x] Add request-side bypass classification before lookup. +- [x] Make unsupported/backend-failed cache lookups fall back to inline processing on non-Fastly + adapters rather than buffering a cacheless ESI path. +- [x] Run focused eligibility tests and commit. + +### Task 5: Move request collapse before the origin fetch + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` + +- [x] Verify the reservation is acquired before origin work and the Fastly transaction contract + blocks same-key waiters. Viceroy is single-threaded, so it cannot directly reproduce two + truly concurrent cold requests. +- [x] Introduce a lookup outcome with an opaque insert reservation and explicit cancellation. +- [x] Implement Fastly `Transaction::lookup` before origin work and consume/cancel its obligation + on every exit path. +- [x] Ensure invalid fresh entries become replaceable rather than causing repeated refetches. +- [x] Run focused Core Cache/Viceroy tests and commit. + +### Task 6: Make privacy and policy-header parity final + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/response_privacy.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` tests + +- [x] Write failing tests for repeated CSP/CSP-Report-Only, omitted COOP/COEP/CORP/HSTS/Link, + unknown cached header metadata, duplicate required metadata fields, and a late integration + changing `Cache-Control` to public. +- [x] Capture all ordered values, expand the safe allowlist, and decode metadata strictly. +- [x] Replay with `append`, then apply the assembled-response privacy policy last. +- [x] Preserve and reassert private/no-store after request-filter effects in Fastly's final send. +- [x] Run focused header/privacy tests and commit. + +### Task 7: Bypass shared templates for request-private diagnostics + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write a failing warm-cache test activated by diagnostics query and another by diagnostics + cookie. +- [x] Make `requires_private_no_store()` a lookup/store disqualifier. +- [x] Verify ordinary diagnostics-disabled requests still hit C2. +- [x] Run focused diagnostics/C2 tests and commit. + +### Task 8: Re-encode assembled responses for the reader + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write failing cold/warm tests requiring gzip/br clients to receive a matching encoded body + and proving reader encoding no longer partitions the stored template. +- [x] Keep the origin offer within the reader's supported codings so a response-gate bypass remains + lossless, while decoding every stored template to identity. +- [x] Carry the selected response encoding separately from identity template metadata. +- [x] Encode buffered assembly after splicing and stream hit prefix/seam/suffix through one encoder. +- [x] Handle `identity;q=0` without serving an unacceptable representation. +- [x] Emit the correct `Vary: Accept-Encoding` response semantics after final encoding. +- [x] Run focused compression tests and commit. + +### Task 9: Make marker failures safe + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` + +- [x] Write failing tests for HTML with no explicit ``, a publisher-authored marker + collision, and a corrupt cached marker. +- [x] Record/validate a schema-bound seam location or use a collision-resistant marker contract. +- [x] Cancel storage and fall back safely when the optimization cannot produce one seam. +- [x] Run focused miss/hit assembly tests and commit. + +### Task 10: Add operational observability and harden the harness + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` +- Modify: `scripts/c2-local-test.sh` +- Modify: `.github/workflows/test.yml` + +- [x] Write failing tests for distinct backend-error versus not-found status and C2 response-state + reporting. +- [x] Preserve backend errors and emit bounded C2 status without exposing key material. +- [x] Change the harness to operate on a temporary manifest and fail on missing/non-numeric probe + output or empty response bodies. +- [x] Test both cold and warm integrity and execute the generated scheduler payload contract. +- [x] Add the ESI harness to CI where Viceroy prerequisites are available. +- [x] Run shell syntax/static checks and commit. + +### Task 11: Document configuration, semantics, and rollback + +**Files:** + +- Modify: `trusted-server.example.toml` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md` +- Modify: `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md` +- Modify: relevant #1009 findings documents + +- [x] Document that `esi` means Fastly C2 plus byte-seam assembly, not parser execution or final + HTTP shared caching. +- [x] Document `template_cache_vary`, cookie independence, freshness, metrics, purge, rollback + ordering, and limitations on non-Fastly adapters. +- [x] Close or supersede stale spike checkboxes and remove claims contradicted by the final code. +- [x] Run docs format/build and commit. + +### Task 12: Full verification + +**Files:** none expected beyond fixes discovered by verification + +- [x] Run `cargo fmt --all -- --check`. +- [x] Run all four adapter test aliases and the parity suite. +- [x] Run all six clippy aliases. +- [x] Build the Fastly release WASM. +- [x] Run JS tests, build, and format under pinned Node 24.12.0. +- [x] Run docs format/build. +- [x] Run `scripts/c2-local-test.sh esi` and `inline` if the environment exposes the required + local certificate store; otherwise report the exact environment blocker. +- [x] Run `git diff --check`, inspect the merge graph, and confirm the worktree contains only + intended changes. + +### Task 13: Interpret Fastly Surrogate-Control conservatively + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Write a failing gate test using `Cache-Control: max-age=60` plus the observed publisher + `Surrogate-Control` policy (`max-age=1200`, `stale-while-revalidate=21600`, and + `stale-if-error=604800`). +- [x] Write failing tests proving the shorter standard/surrogate freshness wins, stale windows do + not extend fresh reuse, restrictive directives are refused, and unknown, duplicate, or + malformed directives fail closed. +- [x] Parse only Fastly's supported `max-age`, `stale-while-revalidate`, and `stale-if-error` + directives; continue refusing every other vendor CDN policy field. +- [x] Keep request `Cache-Control: max-age=0` as an intentional C2 bypass so reload preserves its + revalidation semantics. +- [x] Run focused tests, `cargo test-fastly`, target-matched formatting/clippy, both local harness + modes, and verify the observed publisher policy progresses from `miss-stored` to `hit` in + the local Fastly runtime on an ordinary navigation. + +### Task 14: Allow browser reloads to reuse a fresh ESI template + +Task 14 supersedes Task 13's conservative request `max-age=0` bypass after end-to-end testing +proved that C2 reuses only the neutral template and still creates a new private response and +auction. + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Write a failing end-to-end test proving `Cache-Control: max-age=0` reruns the auction but + does not refetch the reader-neutral publisher template. +- [x] Treat only a valid zero request max age as compatible with C2; continue bypassing positive + or malformed constraints and every explicit revalidation directive. +- [x] Verify the focused tests, formatting, and Fastly clippy, then commit independently. + +### Task 15: Make the ESI template-cache ceiling configurable + +Task 15 supersedes Task 13's shorter-of-standard-and-surrogate rule. The final behavior follows +Fastly edge precedence while retaining restrictive directives as hard refusals. + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `trusted-server.example.toml` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Add failing configuration tests for the 60-second default, an explicit 1,200-second ceiling, + zero, values above one day, and omission from serialized rollback-compatible config. +- [x] Add failing freshness tests proving Fastly precedence, age deduction, and the configured + ceiling for the observed `Cache-Control: max-age=60` plus + `Surrogate-Control: max-age=1200` response. +- [x] Implement `template_cache_max_age_seconds` under `[creative_opportunities]` and thread its + resolved duration into C2 eligibility. +- [x] Remove the Fastly adapter's second hard-coded 60-second cap; the already-authorized + per-entry max age becomes the sole insertion lifetime. +- [x] Update the example and operator guide, without editing the tracked deployment + `fastly.toml`. +- [x] Run focused red/green tests, full adapter tests and clippy gates, documentation checks, and + inspect the final diff with `fastly.toml` excluded. diff --git a/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md b/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md new file mode 100644 index 000000000..a4c04b9f8 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md @@ -0,0 +1,104 @@ +# #1009 ESI Parser Assembly Implementation Plan + +> **Execution note:** Implemented inline in the current checkout, without a worktree or +> subagents, as requested. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Use the repaired ESI parser on authorized cold C2 misses without changing the existing warm-hit streaming behavior. + +**Architecture:** C2 retains the inert schema-v4 seam. Core delegates cold assembly through a platform trait; Fastly converts the seam to one synthetic ESI include and resolves it from the already-collected per-reader script. Parser failure falls back to core's validated byte split, while warm hits continue to stream by byte seam. + +**Tech Stack:** Rust 1.95, `wasm32-wasip1`, Fastly Compute/Viceroy, `stackpop/esi` pinned by Git revision, `error-stack`. + +--- + +### Task 1: Restore a platform assembly boundary + +**Files:** + +- Create: `crates/trusted-server-core/src/platform/template_assembly.rs` +- Modify: `crates/trusted-server-core/src/platform/mod.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Test: `crates/trusted-server-core/src/platform/template_assembly.rs` + +- [x] Add a failing object-safety/default-behavior test for `PlatformTemplateAssembler`. +- [x] Run the focused core test and confirm it fails because the boundary is absent. +- [x] Add the trait, error type, unavailable default, runtime service field, builder method, + accessor, and test support. +- [x] Run the focused tests and confirm they pass. + +### Task 2: Delegate only cold-miss assembly + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` + +- [x] Add a recording assembler to the C2 end-to-end tests. +- [x] Add a test asserting one platform call on a cold miss and no additional call on the + subsequent warm hit. +- [x] Add a test asserting platform failure returns a complete byte-seam response. +- [x] Add tests for `x-ts-assembly` values on parser, fallback, and warm paths. +- [x] Run each test first and confirm the expected failure. +- [x] Change `assemble_if_shared` to call the platform assembler after storage, fall back + to the validated byte split on error, and return the assembly method. +- [x] Set `x-ts-assembly` without changing `x-ts-c2-cache` or privacy headers. +- [x] Re-run the C2 end-to-end test module. + +### Task 3: Add the repaired Fastly ESI adapter + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/Cargo.toml` +- Modify: `Cargo.lock` +- Create: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Test: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` + +- [x] Add failing adapter tests for a large Next.js script followed by the seam, an + unexpected publisher ESI directive, an unexpected dispatcher URL, and verbatim + fragment content. +- [x] Run the focused Fastly test filter and confirm the missing module/implementation + fails. +- [x] Pin `https://github.com/stackpop/esi.git` at + `4c53feab4d22ad9a84641b4c46f3f63bc6d197e2`. +- [x] Implement the explicit no-cache/no-DCA ESI configuration and synthetic completed + fragment dispatcher. +- [x] Register `FastlyTemplateAssembler` in per-request runtime services. +- [x] Run the focused Fastly tests and confirm they pass. + +### Task 4: Preserve cache schema and documentation truth + +**Files:** + +- Modify: `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md` +- Modify: `docs/guide/configuration.md` +- Modify: `scripts/c2-local-test.sh` +- Test: `crates/trusted-server-core/src/platform/template_cache.rs` + +- [x] Add/adjust tests proving schema version 4 and the inert stored marker remain + unchanged. +- [x] Extend the local harness to require `esi-parser` on the miss and `byte-seam` on the + hit. +- [x] Update architecture and operator documentation to describe the hybrid path and + pinned fork accurately. +- [x] Run formatting and the harness's static checks. + +### Task 5: Full verification and signed commit + +**Files:** + +- Review every modified file. + +- [x] Run `cargo fmt --all -- --check`. +- [x] Run every target-matched Clippy alias from `CLAUDE.md`. +- [x] Run `cargo test-fastly`, `cargo test-axum`, `cargo test-cloudflare`, and + `cargo test-spin`. +- [x] Run the integration parity test. +- [x] Run JS tests/build/format and docs format. +- [x] Run the C2 local harness when Viceroy and its certificate environment are + available; otherwise report that environmental gap explicitly. +- [x] Run `git diff --check`, inspect staged scope, and confirm no operator configuration + or secrets are staged. +- [x] Create one SSH-signed commit only after every required gate is green. +- [x] Verify the commit signature locally and report the exact commit ID and test counts. diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md new file mode 100644 index 000000000..031ecc50c --- /dev/null +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -0,0 +1,864 @@ +# The Cacheable Root: Latency Diagnosis and Stage 0 Design + +_Filename retains its original `esi-` prefix; the commit history and every +cross-reference point at it. The subject moved, the path did not._ + +**Issue:** IABTechLab/trusted-server#1009 · **Date:** 2026-08-08 · + +> **HISTORICAL RECORD — NOT THE CURRENT IMPLEMENTATION.** This document preserves the +> measurement and feasibility investigation. Every executable ESI tag, parser, and +> subrequest described below belongs to a rejected spike; do not use those sections to +> infer current runtime behavior. The final branch retains `assembly_mode = "esi"` only as +> the operator spelling for Fastly C2 plus exact byte-seam assembly. See +> [the merge-hardening design](./2026-08-12-1009-esi-merge-hardening-design.md). + +**Revised:** 2026-08-10 +**Baseline:** citations verified at `cfb98f4`; unchanged as of `b0ce56c3`. + +> ## ⚠️ Correction, 2026-08-10 — this document's original ESI verdict was wrong +> +> The first revision concluded that ESI was **structurally blocked**: that it +> presupposed a TS-owned template cache which did not exist, and that such a cache was +> in turn blocked on purge capability the platform did not offer. **Both claims are +> false**, and an external review was right to reject them. +> +> Verified against the pinned `fastly` 0.12.1: +> +> - **The cache boundary is native.** `fastly::cache::core` provides +> `insert(key, max_age).execute() -> StreamingBody` for arbitrary bytes, `lookup()` / +> `found()` to read them back, and `Transaction` with `must_insert()` for request +> collapsing. The two-stage design needs no separate KV or template service. +> - **Purge exists in-process.** `fastly::http::purge::purge_surrogate_key` purges from +> inside Compute; the management-API token scope cited in the original is irrelevant to +> it. Note which cache, though: `InsertBuilder::surrogate_keys([...])` is the **Core +> Cache** API and keys the transformed-template cache (C2). It does **not** key the HTTP +> read-through cache (C1) that Stage 0 turns on — purging that needs origin-supplied +> keys or the HTTP cache's own surrogate-key surface. +> - **The original pipeline ordering was backwards.** It said "order esi → lol*html, +> never the reverse." `lol_html` \_emits* the ESI include tags, so ESI must run after it. +> Correct order is in [§6.6](#66-the-esi-pipeline-corrected). +> +> The error was inspecting what this repository does and reporting it as what the +> platform permits — the same mistake this document criticises #1009 for making in the +> other direction. +> +> **ESI is therefore feasible and unvalidated, not rejected.** Validating it is +> [a separate plan](../plans/2026-08-10-1009-esi-validation-spike.md). What survives +> here is the latency re-diagnosis and the Stage 0 optimisation, which are worth doing +> and are **not** an answer to #1009. + +## Document map — read this first + +#1009 is answered across three documents, not one. This is the only place that says +which owns what. + +| Document | Owns | +| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------- | +| **This spec** | Why the TTFB regression happens, what Stage 0 is and why, and the corrected ESI feasibility verdict | +| [Stage 0 plan](../plans/2026-08-08-1009-measurement-and-stage-0.md) | Implementing the measurement and the cache-bypass flag. **Does not close #1009.** | +| [ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md) | **Where #1009 is actually decided.** Four arms, safety gates, decision rule. | +| [Findings](../plans/2026-08-08-1009-measurement-findings.md) | Recorded results. Currently: Step A only, at `PROVISIONAL PASS`. | + +**If you want the ESI answer**, it is [§2](#2-why--the-three-findings) for the verdict, +[§6.6](#66-the-esi-pipeline-corrected) for the pipeline, and the spike plan for how it +gets validated. Everything else here is Stage 0 and the latency analysis behind it. + +**Decision requested:** approve the four items in §1. + +> **Orientation.** #1009 asks whether Edge Side Includes (ESI) can cache page fragments +> so that cacheable publisher HTML is separated from per-user ad state, recovering a +> TTFB regression that Trusted Server (TS) adds to navigations on a Next.js App Router +> publisher running on Fastly Compute. ESI can do this; whether it should is not settled +> here. Separately, the regression has a cheaper cause than the issue assumes. +> +> **This document deliberately carries no performance measurements.** Every conclusion +> below is derived from code at the pinned baseline, so it can be checked by reading the +> repository rather than by trusting a benchmark. Where a quantity is needed and unknown, +> it is named as unknown and [§3](#3-monday-morning) says how to obtain it. +> +> Terms used throughout: **the hold** = TS holding the HTTP response open at `` +> until the server-side auction (SSAT) resolves. **React #418** = the React +> hydration-mismatch error raised when `adInit()` mutates ad-slot subtrees during +> hydration; it is why bid application is deferred to `window.load`. It is a React error +> number, **not** a repository issue — the tracker is +> [#938](https://github.com/IABTechLab/trusted-server/issues/938). **The SSAT price +> defect** = a live mispricing bug named in #1009 (prices reading 100× high) — cited +> from #1009 and prior investigation, not re-verified here. + +--- + +## 1. Decision requested + +| # | Decision | Owner needed | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| D1 | **ESI is feasible and unvalidated.** Validate it via [the spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), not by deferring it. | Eng + product | +| D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | +| D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to a **`FINAL PASS`** in §3. A `Vary` check alone is a `PROVISIONAL PASS` and is not a release gate. Rollback needs a purge path, not only a config push. | Eng | +| D4 | **Stages 1–2 queue behind the SSAT price defect and #938.** Stages 3b–4 unscheduled. ESI is not in this queue — see §7. | Product | + +Rationale for D4 in [§8](#8-priority). Everything this document recommends _against_ +doing is in [§7](#7-deferred-work-specified-not-scheduled), at deliberately lower +detail than the work it recommends. + +--- + +## 2. Why — the three findings + +**ESI is buildable on the pinned SDK, and unvalidated.** `lol_html` emits executable ESI +include tags into a shared template; `fastly::cache::core` stores that template; the +`esi` crate assembles per request on the way out. Everything that requires is +already a dependency. The real open questions are empirical, not architectural: does it +beat a plain client fetch by enough to justify a Fastly-only rendering path, and can +per-user leakage be excluded under cold MISS, warm HIT, stale revalidation, and fragment +failure. [The spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) answers +those; [§6.6](#66-the-esi-pipeline-corrected) gives the pipeline. + +Two constraints stay true regardless. ESI is **Fastly-only at every API level**, so it +is a per-platform accelerator rather than the architecture, and its maintenance cost +belongs in the decision. And its Dynamic Content Assembly must be **explicitly disabled** +— bid payloads carry partner-controlled creative markup, so under `DcaMode::Esi` an SSP +could embed an ESI include targeting an arbitrary URL and make the edge fetch it. Details in +[Appendix E](#appendix-e--esi-notes-condensed). + +**The auction is already out of band; the hold is ~free.** It is dispatched _before_ +the origin fetch and does not block — dispatched at `publisher.rs:2751-2755`, sent at `:2870` — +with a 500 ms budget. The actual cost is `with_cache_bypass` +(`publisher.rs:2867`), +which forces every ad-eligible navigation to miss the Fastly readthrough cache. + +**The two fixes are multiplicative.** Removing the bypass alone lets the previously +hidden auction surface as the new bottleneck. Removing the hold alone changes nothing, +because the auction was never the bottleneck. **Shipping the hold removal without the +bypass removal will measure no improvement and will read as the effort having failed** — +the most likely way this work gets judged unfairly. + +**Ordering is established; magnitude is not.** The ordering above follows from code and +needs no measurement. The _size_ of the win does — and the one quantity it depends on, +the origin build time under `Pass`, has never been measured. #1009's timings do not +supply it: they compare cached fetches against each other, not against an origin build. +**Quote no figure to a publisher until §3 Step C runs.** Full reasoning in +[§6](#6-the-analysis). + +--- + +## 3. Monday morning + +Three checks, ordered cheapest-first. Each needs a named owner before starting. + +**Step A — origin `Vary` and cookie check (minutes for the first pass).** `curl` the +origin with and without `RSC`, `Next-Router-*`, and the experiment header; inspect `Vary`, +`Cache-Control`, and `Set-Cookie`. **This first pass yields a `PROVISIONAL PASS` only** — +it is not what gates the flip. A `FINAL PASS` additionally requires a real authenticated +session, Basic Auth through TS, the experiment variant, representative routes, and +cached-hit render attribution. Do the cheap pass first because it is the +cheapest thing that unblocks anything. + +**Step B — what consumes TS's own response headers (under a day).** Request a TS-served +path that already emits `public, s-maxage` +(`http_util.rs:294-311`) +twice and look for `x-cache`/`age` on TS's _own_ response. **Gates the Stage 3a/3b +split** — see [§7](#7-deferred-work-specified-not-scheduled). + +**Step C — measure the hold directly (1 day + a measurement window).** + +The hold's cost is literally the duration of one `.await`: `collect_stream_auction` at +`publisher.rs:793`, plus the +two EOF variants in `hold_finish_ready_segments` and `hold_finish_tail_segments`. Two +`Instant`s around it yield **`hold_wait_ms`** — the number this entire document is +arguing about, measured rather than modelled. + +Emit two timings per ad-eligible navigation: + +| Metric | Why | +| ----------------- | ---------------------------------------------------------------------- | +| `hold_wait_ms` | **The decision.** How long the response was actually held for bids. | +| `origin_fetch_ms` | Attribution — how much of the win Stage 0 can claim. Origin TTFB only. | + +`hold_wait_ms` replaces the proxy comparison an earlier draft proposed. Comparing `O` +against `A` was an indirect way of asking "does the hold block?"; this asks it directly, +costs less to build, and removes the modelling error corrected in +[§6.2](#62-what-the-hold-actually-costs). + +Deliberately not measured: auction collect duration is already instrumented +(`OrchestrationResult::total_time_ms`, `auction/orchestrator.rs:285`, flowing to +`auction_events_raw`) — read it, don't rebuild it. Rewrite duration decides nothing and +would mean touching two finalizers. + +- **Mechanism: a `log::info!` line behind a debug flag, not `Server-Timing`.** A response + header would in fact work for the origin-fetch figure — that value is known before + headers commit — but a server-side log needs no browser harness to collect it, `log` is + this project's instrumentation crate, and the auction path already measures itself with + `web_time::Instant`. Gate it behind config: one line per eligible navigation is real log + spend and the instrumentation is temporary. +- **Sample: enough navigations per arm to separate the medians with confidence**, across + both page types, and state the N alongside any result. #1009's sample was small enough + that its conclusion did not survive contact with the code; replacing it with another + underpowered sample would repeat the error. + +**Step C has two outcomes, both actionable:** + +| `hold_wait_ms` median | Meaning | Effect on staging | +| --------------------- | ----------------------- | ------------------------------------------------------------ | +| Near zero | The hold is free | Proceed as staged: Stage 0 primary, Stage 2 protects its win | +| Materially non-zero | The hold **is** costing | **Staging inverts** — Stage 2 primary, Stage 0 secondary | + +The work does not change; its order and justification do. **The staging in §7 is +conditional on this measurement**, and the second outcome is a live possibility rather +than a formality — §6.2's argument for the first is weaker than an earlier draft claimed. + +Stage 1's bids-fetch timeout still needs a measured client-side figure rather than an +invented constant, but Step C is server-side and does not supply it. Capture it from the +browser harness when Stage 1 is actually scheduled. + +--- + +## 4. Stage 0 — the only build item recommended now + +Stop bypassing the read-through cache on ad-eligible navigations +(`publisher.rs:2867`). + +**Ship it as an operator flag, not a deletion.** Add +`publisher.bypass_origin_cache`, defaulting to today's behaviour, in the same release as +the Step C instrumentation. Then turn it off with `ts config push`. + +The diff is slightly larger than deleting a line, and that is the point. The risk being +gated here is **cache poisoning** — serving one representation in response to a request +for another. For that class of failure, rollback speed dominates diff size: a config push +reverts the read path in seconds where a release does not — but a config push **evicts +nothing**, so full rollback is flip, then purge or roll a versioned key namespace, then +observe past the origin TTL. The flag also buys an A/B on a byte-identical +build, removing build difference as a confound in the very measurement this depends on, +and allows flipping for a tester-cookie population before all traffic. + +Retire the flag once the change has held: flip the default, then delete the setting and +its branch. A temporary flag left in place becomes permanent configuration surface. + +### What to watch after the flip + +Two regression signals, both checked before the win is: + +- **`unexpected_origin_304` abandonment rate.** That reason + (`publisher.rs:2894-2916`, + emitted via `emit_abandoned_auction` at `:2360`) exists precisely because the ad-stack + path refuses cached and conditional origin responses. Re-enabling the cache is what + could revive it. **Any non-zero rate is a rollback signal** — it means a 304 is reaching + TS that the conditional-header strip was supposed to make impossible. +- **Representation mixing.** Spot-check that HTML navigations still return HTML and RSC + fetches still return `text/x-component`. A mismatch means the `Vary` risk materialized + despite a PASS verdict. Roll back immediately; this is cache poisoning, not a + performance regression. + +**Why it is safe in principle.** The conditional-header strip runs 34 lines earlier +under the same gate (`publisher.rs:2832-2836`, +which also strips `Range`/`If-Range`), so the request already reaches the cache +unconditional and a HIT returns a full body. [The 304-prevention design](./2026-07-22-ssat-root-document-304-prevention-design.md) +added the bypass as belt-and-braces and listed the TTFB cost under its own Risks. The +strip alone satisfies its invariant. + +**But it carries a risk that design never considered — and this is the blocking +precondition.** RSC fetches are not navigations +(`is_navigation_request` +requires `Sec-Fetch-Dest: document`), so they never set the bypass and **already flow +through the readthrough cache**, while HTML navigations are `PASS`. Removing the bypass +puts both representations under one cache key. #1009 states the origin varies on +`rsc`, `next-router-*`, and a publisher-specific experiment header — if that variance is +not declared via `Vary`, the +cache can serve a flight payload to an HTML navigation. + +The classification is also not airtight: `is_navigation_request` falls back to the +`Accept` header when Fetch Metadata is absent, and its own comment warns _"this path is +weaker — `fetch()` can set Accept: text/html"_ +(`http_util.rs:84-88`). + +**A FAIL is not merely a Stage 0 blocker — it is a live production defect.** RSC fetches +already transit the read-through cache today, because they never set the bypass. If the +origin varies on `Next-Router-*` without declaring it, TS is cross-serving RSC variants +right now. On a FAIL, file that immediately and treat "ask the origin to declare `Vary`" +as urgent rather than as the cheaper of two options. + +**The `Vary` check is necessary but not sufficient.** Turning the read-through cache on +for HTML navigations exposes three things a representation check does not cover, and all +three are a larger class than the RSC split: + +- **Client `Cookie`.** TS forwards client cookies to origin unchanged — there is no + `COOKIE` strip on the publisher path. Any cookie-personalized HTML (logged-in state, + paywall meter, publisher-side A/B assignment) becomes cross-servable unless the origin + declares `Vary: Cookie` or marks those responses private. +- **Origin `Set-Cookie`.** If the origin emits `Set-Cookie` alongside a shared-cacheable + `Cache-Control`, the read-through cache can replay one visitor's cookie to the next. + TS's own privacy net downgrades **TS's** response — it runs after the cache has already + stored the origin's. +- **`Authorization`.** #1009 describes a basic-auth-gated deployment. Responses to + authorized requests entering a shared cache needs its own check. + +So Step A must capture `Cache-Control` and `Set-Cookie` too, and repeat each request with +and without a session cookie. Same minutes of work; closes the bigger hole. + +**Two effort branches, and Step A's `Vary` result decides which** — note this selects the +_shape_ of Stage 0, while the `FINAL PASS` conditions decide _whether it ships at all_: + +| Step A result | Stage 0 is… | Effort | +| ---------------------- | --------------------------------------------- | ------ | +| Origin declares `Vary` | the flag, its tests, then a config push | 1–2 d | +| Origin does **not** | a TS-side cache-key discriminator — a feature | 4–8 d | + +The discriminator is the safer design either way, because it keys on the headers that +actually distinguish the representations rather than on the navigation classification. + +**Two benefits beyond TTFB, worth stating to a publisher:** + +- **Origin load drops.** The 304-prevention design explicitly accepted _"increasing + origin load"_ as a cost. This reverses it. +- **`stale-if-error` becomes reachable.** Under `Pass` an origin outage is a hard + failure. This needs a decision rather than a default: stale HTML carries stale slot + markup, and whether that beats an error is a product call. + +--- + +## 5. The trap in the deferred work — read this before scheduling Stages 1–2 + +The hold is load-bearing for something other than latency. The invariant is: + +> `ad_bids_state` must be `Some(..)` when `lol_html` processes the `` end tag. + +The end-tag handler (`html_processor.rs:381-395`) +locks that mutex once and falls back to `build_empty_bids_script()` on `None`. + +**Removing the hold without relocating collection renders a normal page with +`tsjs.bids = {}` and no server-side ads** — no error, no non-2xx, no ERROR log. On +Axum, Cloudflare, and Spin the loss is fully silent: +`publisher.rs:2248` holds a +bare `Option` with no guard, so not even a drop warning fires. **The +SSPs are billed regardless.** + +This is why Stage 2 is gated on three companions and a production soak, and why slot +fill cannot be the canary — see [§7](#7-deferred-work-specified-not-scheduled). + +--- + +## 6. The analysis + +### 6.1 Corrections to #1009's premises + +| # | #1009 states | Verified against `cfb98f4` | +| --- | -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Two per-user injection seams | Partly. `tsjs.adSlots` **content** is per-URL — `build_slot_json` emits config- and path-derived fields only. But its **presence** is gated on `should_run_ad_stack` (consent, bot, prefetch, kill switch), so it is request-dependent and **must not live in a shared template**. See §6.7. | +| 2 | Identity off-inline is a prerequisite | Privacy net is cookie-gated and returning navs set no cookie (`ec/finalize.rs:86-94`). **First-visit only** — but note the corollary: because returning navigations set no cookie, that net never fires for them and is **not** a backstop against shared-caching a per-user response. | +| 3 | Stamp at `:2882-2888` | `:2945-2963`, `private, no-store`, also removing `ETag`/`Last-Modified`/four CDN headers. **Seven headers.** | +| 4 | Three cacheability killers | Two more: `bypass_cache` and the `304→502 guard`. **The bypass is the cost.** | +| 5 | Two `!Send` pipelines is the hard part | `?Send` already pervasive. **Not the obstacle.** | +| 6 | Goal: root as a shared Fastly HIT | Nothing caches TS's own response on Compute; the A/B's `x-cache` is the **backend readthrough** cache. **Reframes the goal.** | + +Rows on #1009's `esi` compatibility check (holds), its drifted line numbers, and its +two broken `#1`/`#3` cross-references are in [Appendix A](#appendix-a--full-1009-correction-table). + +**Credit where due.** #1009 names the hold as blocker 1 and states it correctly. What +changes here is its _causal weight_. Likewise, #1009's own observation that TS _"shifts +the auction cost from client-side to server-side rather than adding new work"_ is the +argument for client-fill, which the issue then declines in favour of ESI. + +### 6.2 What the hold actually costs + +**An earlier draft of this section claimed a stronger argument than the code supports. +It was wrong, and the correction matters.** + +The hold does not key off `lol_html` at all. `BodyCloseHoldBuffer::push` +(`publisher.rs:2190-2202`) +scans the **decoded origin input** for ` Dispatch precedes the origin fetch, so the hold costs `max(0, A − T)`, where `A` is the +> auction collect duration and `T` is origin TTFB plus body transfer up to the `` +> byte. Since `` sits at the end of a document, `T` is close to the full download. + +`A` is bounded by `auction_timeout_ms`, resolved as +`creative_opportunities.auction_timeout_ms` falling back to `auction.timeout_ms` +(`publisher.rs:2680-2684`) +— check the resolution order against your own config rather than trusting a number; the +shipped example sets different values at each level. + +**This is a claim requiring measurement, not a proof.** §3 Step C measures the hold's +cost directly rather than inferring it. + +A finding that does survive, and belongs with [the ceiling](#64-the-ceiling): because +`HtmlWithPostProcessing` withholds all output until the final chunk, the streaming-prefix +design at `publisher.rs:1343-1348` +— whose comment promises "the client receives the document up to `` while the +auction rides alongside transfer" — is **inert on a Next.js publisher**. Every +`step.ready` yields empty bytes. That comment is misleading on exactly the publisher +under discussion. + +### 6.3 The quantity nobody has measured + +Write the fetch time under `Pass` as `O`. Recovery depends on it, and it has never been +captured. #1009's timings cannot supply it: they compare a POP hit against a +shield-served fetch, both of which are _cached_ paths, whereas `CacheOverride::Pass` +bypasses TS's read-through cache and its shield. + +Note `Pass` bypasses **TS's** caches only. It has no authority over any CDN the publisher +runs in front of their own origin — and #1009's `x-cache: MISS, MISS` on the TS-on arm +hints one may exist. So `O` may not be origin build time at all. Since `O` is the single +quantity this model depends on, that ambiguity is worth resolving in Step C rather than +assuming. + +What follows from code alone, without any number: + +| Configuration | Long pole after the change | Recovery | +| ------------------- | -------------------------- | ------------------------------ | +| Hold removal only | origin (still `PASS`) | **none** | +| Bypass removal only | the auction budget | partial — the auction surfaces | +| **Both** | the rewrite | **the full available win** | + +That ordering is what the staging rests on, and it is measurement-independent. The +magnitude of each row is not, and §3 Step C supplies it. + +### 6.4 The ceiling + +#1009 targets "approach the TS-off warm numbers." **Unreachable, structurally.** Those +numbers are TS-off _streaming_ a POP HIT. TS buffers the whole document before emitting +a byte (16 MB cap), so its floor is `full origin body download + full rewrite` — above a +streamed hit by construction, whatever the timings turn out to be. Set the target from +Step C's measured rewrite cost rather than from the TS-off baseline. Going below the +floor requires true origin streaming (#849), out of scope. A non-Next.js publisher with +no post-processor takes the streaming path and would see a lower floor. + +### 6.5 Confidence + +**High on the structural claims.** §6.2's argument, the bypass forcing a cache miss, the +the silent-empty-bids failure mode, the geo and `Vary` blockers, +and the fill-canary blindness are all read directly out of the code at `cfb98f4`. Anyone +can check them without running anything. + +**None on magnitude.** `O` is unmeasured and the rewrite cost is unmeasured. This +document does not estimate them, and no figure in it should be quoted as one. + +Worth stating plainly: #1009 reached the opposite causal conclusion from a small sample. +That is a caution about small samples generally, not only about that one — which is why +§3 Step C specifies the measurement rather than this document supplying a substitute +for it. + +### 6.6 The ESI pipeline, corrected + +An earlier revision of this document said "order esi → lol*html, never the reverse." +That is backwards. `lol_html` is what \_emits* the ESI include tags; ESI cannot process +tags that do not exist yet. The correct order: + +``` +origin → lol_html transform → fastly::cache::core → finalize headers → stream esi assembly → client + (one unconditional marker (shared template, (EC cookie, geo, (per request, + at the body-close seam; surrogate-keyed, unconditional fetch the + the head seam is NOT a TS-chosen TTL) private/no-store) fragment) + hole — adSlots presence + is request-gated, §6.7) nothing may change + after this point +``` + +The push/pull mismatch that the earlier revision treated as a blocker is real but +irrelevant: `lol_html` pushes, `esi` pulls, and **the cache is the buffer between them**. +That is not an obstacle to the two-stage design — it _is_ the two-stage design, which is +what #1009 proposed in the first place. + +Mechanism, all present in the pinned `fastly` 0.12.1: + +| Need | API | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Store the template | `cache::core::insert(key, max_age).execute() -> StreamingBody` | +| Read it back | `cache::core::lookup(key)` → `found()` | +| Avoid a thundering herd | `cache::core::Transaction` — `must_insert()` / `must_insert_or_update()` | +| Invalidate **C2 only** | `InsertBuilder::surrogate_keys([...])` (Core Cache) + `fastly::http::purge::purge_surrogate_key`. Does **not** key C1 — see the row below. | +| Invalidate C1 | Origin-supplied surrogate keys, or the HTTP cache's own surrogate-key surface. Not the Core Cache API. | + +Purge runs **inside Compute**. The management-API token scope cited under +[Stage 4](#7-deferred-work-specified-not-scheduled) governs a different surface and does +not gate this. + +**Three caches, kept distinct.** Conflating them is what produced the original error: + +1. **Origin read-through** — raw origin bytes. What Stage 0 turns back on. +2. **Shared transformed template** — post-`lol_html`, pre-ESI, no per-user data. The ESI + target, and new. +3. **Assembled-response delivery cache** — the final per-user output. **Must never + exist.** Nothing in this document or the spike proposes one. + +**Validation constraint.** Viceroy 0.17 cannot exercise the customized read-through hooks +end to end. Unit tests can cover the transform and the security properties; MISS / HIT / +stale / shielding behaviour must run against a real Fastly test service. + +--- + +### 6.7 What may and may not live in a shared template + +A correction to §6.1 row 1, and the constraint that governs any shared-template design. + +The original framing — "`adSlots` is per-URL, so there is one per-user hole, not two" — +is half right and dangerously so. `build_slot_json` really does emit only config- and +path-derived fields. But whether the script is emitted **at all** is gated on +`should_run_ad_stack` (`publisher.rs:2920-2927`), which is +`is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the _content_ is per-URL and the _presence_ is per-request. A shared object filled by +the first request would freeze that request's consent decision, bot classification, +prefetch status, and kill-switch state for every later reader. A consent-denied fill +serves a no-ads template to consenting users; a consenting fill serves ad markup to +someone who refused. + +**The rule for anything cached and shared:** + +| May live in the template | Must live in the per-request fragment | +| --------------------------------------- | ---------------------------------------------- | +| tsjs bundle script tag (content-hashed) | `tsjs.adSlots` — presence is request-gated | +| URL rewrites (per-host, in the key) | `tsjs.bids` | +| | GPT diagnostics bootstrap (cookie/query-gated) | +| | Integration head-inserts (request-scoped) | + +The test that catches this class is **byte-identity of the template across requests +differing in consent, bot classification, and prefetch status** — not an absence-of- +per-user-values scan, which the broken design would have passed. + +This applies to any shared-template work, ESI or client-fill alike. The +[spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) implements it. + +--- + +## 7. Deferred work, specified not scheduled + +**The full sequence, in one place.** Stage 0 is specified in [§4](#4-stage-0--the-only-build-item-recommended-now) +rather than repeated here; everything below it is deferred. + +| Stage | What | Status | +| ----- | ----------------------------------------------- | ---------------------------------------------- | +| **0** | Operator flag disabling the origin cache bypass | Recommended now. Gated on a `FINAL PASS`. §4. | +| 1 | Bid delivery off the response body | Deferred behind the correctness defects | +| 2 | Delete the `` hold | Deferred; one-way, needs a Stage 1 soak | +| 3a | Browser caching (`private, max-age` + `ETag`) | Specified, low risk, unscheduled | +| 3b | Shared cacheability | Blocked on geo suppression, `Vary`, and Step B | +| 4 | Purge wiring | Prerequisite for any TS-owned cache | + +**ESI is not a stage here.** It was Stage 5 in an earlier revision, queued behind the +rest. It no longer queues: it is feasible on the pinned SDK and is decided by +[its own spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), which runs +independently of Stages 1–4. The shared template cache it needs is `fastly::cache::core` +([§6.6](#66-the-esi-pipeline-corrected)), not a new service. + +Lower detail below is deliberate. Full specifications are in the appendices. + +**Stage 1 — bid delivery off the response body.** The client fetches `/_ts/page-bids` at +navigation generation 0. Endpoint, same-origin gate, wire shape, and client consumer +already exist. Three decisions must be made before planning: the `slots: []` precedence +rule when head-open already injected a non-empty `ts.adSlots`; the new terminal-event +emission point; and whether the dispatch/collect split survives at all. Plumbing detail +in [Appendix B](#appendix-b--stage-1-plumbing-condensed). Estimated 8–13 d, low-to-medium +confidence, uncertainty concentrated client-side. + +Three companions are mandatory, not optional: **suppress the server bids script +entirely** (not an empty one), **fail loud** (the end-tag handler takes bids by value so +a missing auction is a compile error), and **relocate telemetry** (navigation +`Completed` rows are emitted only from the collect functions, and the `ts-debug` dump +rides the same string). Behaviour change to accept: under client-fill the auction runs +only if the browser executes the fetch, so bots and JS-disabled clients stop triggering +server-side auctions — revenue-relevant, sign unknown. + +**Stage 2 — delete the hold.** 5–8 d. **Rollback is one-way**: it deletes the hold, the +dispatch/collect split, and twelve tests, so the only revert is a release. Ships only +after Stage 1 has run flag-on in production for a window defined _before_ Stage 1 +starts, with TS-attributed renders flat and `auction_events_raw` navigation rows intact. +Secondary wins: removes the duplicated per-codec decoder/encoder wiring, six compression +imports, and the non-parser-context `` runs _all_ attempts and concatenates every non-failed output** — not + first-success-wins, so primary/fallback pairs render both. Least obvious behaviour in + the crate. +- Single include, not per-slot: the auction is one operation producing all slots' bids. + +--- + +## Appendix F — deferred open items (condensed) + +Implementation-level, for unscheduled work only. Decisions needing a human are in +[§9](#9-decisions-needed-from-this-review). + +Should `collect_non_html_auction` (`publisher.rs:2388`) go with the hold or stay? Is +`body_close_hold_loop_stream` (`:2109`, no production caller) safe to delete, or is the +buffered-adapter streaming cutover (#495) still live? Does hidden-tab rAF behaviour +interact badly with a bids timeout? What are Fastly's pending-request semantics when a +`DispatchedAuction` drops mid-flight? Does `stale-if-error` on a cached root serve +acceptable content given stale slot markup? And the googletag shim discards listeners +queued before it loads (#1009 Part 1) — not filed, should be. + +--- + +## Appendix G — code-grounded seams + +All pinned to `cfb98f4`. + +| Concern | Location | +| --------------------------------------------------------- | ------------------------------------------------------------------------------- | +| Eligibility decision | `publisher.rs:2651`, `:2660` | +| `is_navigation_request` | `http_util.rs:73-98` | +| Auction dispatch (pre-origin, non-blocking) | `publisher.rs:2698-2760` | +| Auction overlap intent | `auction/orchestrator.rs:950-952` | +| Auction timeout resolution | `publisher.rs:2680-2684` (creative_opportunities, else auction.timeout_ms) | +| Conditional/range header strip | `publisher.rs:2832-2836` | +| Origin cache bypass | `publisher.rs:2866-2868` | +| Origin 304 → 502 guard | `publisher.rs:2894-2916` | +| `adSlots` build (content per-URL, presence request-gated) | `publisher.rs:2920`, `:3558-3577`, `:3501-3525` | +| Uncacheable stamp | `publisher.rs:2945-2963` | +| `` hold — sync / async / Fastly lazy | `publisher.rs:2235` / `:2109` / `:1318-1390` | +| Hold buffer | `publisher.rs:2177-2218` | +| Auction collect (HTML / non-HTML) | `publisher.rs:2431` (emits `:2456`) / `:2388` (emits `:2410`) | +| Abandonment emitter | `publisher.rs:2360` | +| Bids script build | `publisher.rs:3438-3491` | +| `/_ts/page-bids` | `publisher.rs:3611`, handler `:3723`, auction `:3903` | +| Injection seams (head / body-close) | `html_processor.rs:310-363` / `:381-395` | +| Post-processor buffering | `html_processor.rs:62-94` | +| Next.js post-processor registration | `integrations/nextjs/mod.rs:107` | +| Max buffered body (16 MB) | `settings.rs:77-79` | +| EC cookie issuance policy | `ec/finalize.rs:86-107` | +| Cookie-privacy net | `response_privacy.rs:20-61` | +| Geo response headers | `adapter-fastly/src/middleware.rs:194-200` | +| Cacheable-header precedent | `http_util.rs:294-311` | +| Management token lacks purge (wrong surface) | `adapter-fastly/src/management_api.rs:12` | +| In-process purge / surrogate keys | `fastly` 0.12.1 `cache::core`, `http::purge::purge_surrogate_key` | +| Client initial-ad gate | `js/lib/src/integrations/gpt/index.ts:536-555` | +| `adInit` bid application | `js/lib/src/integrations/gpt/index.ts:566`, `:652`, `:657-661` | +| Client SPA auction hook | `js/lib/src/integrations/gpt/index.ts:806`, `:859`, `:892-949` | +| Prior design that introduced the killers | `docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md` | diff --git a/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md new file mode 100644 index 000000000..5b593a15d --- /dev/null +++ b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md @@ -0,0 +1,260 @@ +# Streaming assembly: the architecture #1009 actually needs + +**Date:** 2026-08-11 +**Status:** Decision record. Supersedes the delivery half of the +[ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md); the cache half +stands. +**Issue:** IABTechLab/trusted-server#1009 + +> **Implementation update, 2026-08-14.** Warm C2 hits still use Design C exactly as +> specified here. Authorized cold misses now also validate the repaired +> `stackpop/esi` parser, pinned by commit: the inert C2 marker becomes one synthetic +> include only in a private working copy, resolved from the already-built reader +> fragment without an HTTP request. Parser failure falls back to the byte seam. See +> [the hybrid implementation design](./2026-08-14-1009-esi-parser-assembly-design.md). +> Sections 2–2c and Design B remain investigation history; the native self-subrequest +> design is still not implemented. + +--- + +## 1. The correction this document exists for + +An earlier reading of the latency, recorded in +[the measurement findings](../plans/2026-08-08-1009-measurement-findings.md), said the +`` hold costs approximately nothing and the whole cost is the origin-cache bypass. + +That is true **today, and only today.** It is true for a reason that stops holding the +moment the rest of this work lands: + +| | Origin fetch | Auction | Reader waits | +| ----------------------------------- | ------------ | -------------------- | ------------ | +| Today | ~650 ms | hidden inside it | ~650–800 ms | +| Cached root, **buffered** assembly | 0 | fully exposed | ~auction cap | +| Cached root, **streaming** assembly | 0 | overlapped with send | ~ms | + +The auction is dispatched before the origin fetch and both run concurrently, so the hold +costs `max(0, auction − origin)` — zero while the origin is slow. Make the root cacheable +and the origin fetch disappears; the auction then has nothing left to hide behind and +becomes the _entire_ remaining cost. + +**So the two problems are coupled, and neither fix shows a win alone.** That is why the +issue is right to treat both as prerequisites, and why measuring one at a time misleads. + +Two distinct problems get bundled in the issue as one blocker. They need different fixes: + +1. **Bids live in the response body** → the page is _uncacheable_. Fixed by templatizing. + **Done.** +2. **The response is held for the auction** → the page is _slow_. Fixed by streaming the + shell and filling the seam late. **Not done** — this document. + +## 2. What the current implementation gets wrong + +On a C2 hit, `collect_and_assemble_cached_template` awaits the auction, then assembles, +then returns a fully buffered `PublisherResponse::Buffered`. The reader receives nothing +until bids resolve. + +That relocates the hold rather than removing it, and on a hit it is _worse than today_ in +one respect: there is no origin fetch left to hide it behind, so the full auction latency +lands on first byte. + +The routing decision that caused it — shared modes take the buffered finalizer — was made +because **a store needs complete transformed bytes.** True on a miss. Irrelevant on a hit, +where the template is already materialized. + +## 2b. Demonstrated, not argued + +Run locally under `viceroy serve` against a stub origin with a **self-imposed 1.5 s bid +endpoint**. These are synthetic numbers from a delay chosen to be observable — not a +measurement of any real deployment, and not comparable to publisher data. + +| Request | Cache | TTFB | Total | Origin fetched | +| ------- | ----- | --------- | --------- | -------------- | +| 1 | miss | ~injected | ~injected | yes | +| 2 | hit | ~injected | ~injected | **no** | +| 3 | hit | ~injected | ~injected | **no** | + +Two things are visible, and both matter more than the absolute values: + +1. **The cache works.** One origin fetch across three requests; the C2 log shows one + miss, one store, two hits. +2. **The reader waits exactly as long anyway.** Time-to-first-byte equals total on every + request, so nothing streams — the entire response lands at once, after the auction. + On the hits the origin fetch is gone and first byte still tracks the injected bid + delay. + +That is the claim in §1 and §2 reproduced on demand: a cached root delivers **no latency +benefit to the reader** while the response is held for the auction. It also gives the +harness a pass/fail shape for the change this document proposes — under streaming +assembly, TTFB must fall away from total by approximately the injected delay. + +## 2c. Measured against the shipped path — a ~100x TTFB regression + +`scripts/c2-local-test.sh` runs both modes against the same stub, with a self-imposed +1.5 s bid endpoint. Synthetic numbers, not a measurement of any deployment. + +| Mode | TTFB | Total | +| ------------------------- | ----------------- | ------- | +| `inline` (shipped) | **0.010–0.019 s** | ~1.51 s | +| `esi` (buffered assembly) | **1.524–1.532 s** | ~1.53 s | + +**The shipped path already streams correctly.** First byte in ~10 ms; the article paints +while the auction runs; only `` waits. Buffered assembly turns that into a wait +for the whole auction before the first byte — roughly **100x worse TTFB than doing +nothing**. + +This corrects §1 and §2, which framed buffered assembly as capturing the origin-fetch +saving and merely failing to add the streaming benefit. It is worse than that: it +**removes** a benefit today's code already delivers. The origin-fetch saving is +irrelevant beside losing the stream. + +It also sharpens where production's latency actually goes. Locally the stub origin +answers in ~2 ms, so `inline` TTFB is ~10 ms. In production the origin fetch is slow and +uncached, and TS cannot send a first byte until the origin sends one — so production TTFB +is the **origin fetch**, with the auction hidden behind the remainder of the body plus the +`` hold. The fix is therefore a fast origin _while keeping the stream_: exactly +Design C, and exactly what buffered assembly gives up. + +**Consequence for the plan:** `esi` mode must not be exposed to any traffic in its current +form. It is not a smaller win than hoped, it is a regression. + +### A harness bug worth recording + +The first version of this comparison reported `inline` fetching the origin zero times — +nonsense that still printed four passes. Viceroy was launched inside a subshell, so `$!` +was the subshell rather than the server; cleanup killed the wrapper and orphaned viceroy. +The next run then failed to bind and **silently answered from the previous run's process**, +carrying that run's config and warm cache. + +A harness that answers from the wrong server is worse than one that crashes, because its +output looks like data. Fixed with no subshell, a pre-flight port check, and a startup +wait that fails loudly. The regression above was invisible until the control worked. + +## 3. The decisive facts + +Three, all verified in the codebase rather than assumed: + +1. **The existing streaming path already implements stream-then-stall-at-the-seam.** + `publisher.rs` builds an `async_stream::try_stream!` that streams body chunks and holds + **only** at `` for the auction (`hold_auction`, `AuctionHoldState`). This is + shipping behaviour, not new work. +2. **`EdgeBody::Stream` is an async stream** — consumers call `stream.next().await` — so an + `await` may sit between chunks. Nothing needs a nested executor. +3. **`BodyCloseInjection::Marker(String)` already exists**, and the streaming finalizers + already strip `Content-Length`. + +## 4. Three designs + +| | Streams | Auctions | Requires | Adapters | +| ----------------------------------- | ------- | -------------- | ------------------------ | --------- | +| **A** — buffered assembly (current) | No | 1 | nothing | Fastly | +| **B** — native ESI subrequest | Yes | 1, in fragment | self-referencing backend | Fastly | +| **C** — cached shell + seam split | Yes | 1 | nothing | **All 4** | + +### Design B, for the record + +`PendingFragmentContent::PendingRequest` is what the `esi` crate is built for: the +dispatcher fires a real subrequest and the processor blocks on the handle. Fastly's +`send_async`/`wait` is **synchronous**, so this sidesteps the sync-dispatcher problem +without any executor. + +It also vindicates the _original_ dispatch gate. Under B the root must **not** dispatch, +because the fragment request runs the auction. The later reversal to +`root_auction_is_useful(Esi) = true` is correct for buffered assembly and wrong for +streaming. **Dispatch-usefulness is a function of the delivery mechanism**, which is the +non-obvious coupling in this design space. + +### Design C — the recommendation + +The template carries an **inert HTML comment sentinel** where the reader's ad slots and +bids go, emitted by the existing `Marker` variant: + +``` + +``` + +On a C2 hit: + +``` +commit headers (private, no-store; no Content-Length) ← must precede any byte on Fastly +stream template[..sentinel] ← the article paints here +await the auction ← the only stall, at the very end +write the bids script +stream template[sentinel+len..] +``` + +Since a hit has the whole template in hand, this is a `split_once`, not a streaming +search. Three yields from a `try_stream!`. + +**Why a comment sentinel rather than a byte offset in metadata.** An offset is O(1), but +capturing it means plumbing the writer position into a `lol_html` end-tag handler, and it +does not survive re-encoding. A `find` over a ~100 KB buffered template is free by +comparison. + +**Why a comment rather than executable ESI markup.** An HTML comment is inert. If +assembly ever fails to substitute, the reader sees nothing; an unresolved ESI include +tag renders as visible text. Failure degrades to "no ads" instead of "broken page". + +**Why not re-run `lol_html` over the cached template.** It would inject a second tsjs +`" + ), + format!("", GPT_BOOTSTRAP_JS), + ]; + ``` + + Verify the false string remains exactly: + + ```text + + ``` + +- [ ] **Step 5: Override the metadata hook from the same `GptConfig`.** + + ```rust + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + if self.config.gam_attribution_enabled { + vec![("data-ts-gam-attribution", "true")] + } else { + Vec::new() + } + } + ``` + + Do not store this state in `HtmlProcessorConfig` or + `IntegrationDocumentState`. + +- [ ] **Step 6: Run focused and neighboring GPT tests.** + + ```bash + cargo test-fastly gam_attribution + cargo test-fastly head_injector + ``` + + Expected: PASS; false preserves two current inserts, true adds the flag and + metadata while still emitting two inserts when `slim_prebid_url` is absent. + +- [ ] **Step 7: Commit.** + + ```bash + git add crates/trusted-server-core/src/integrations/gpt.rs crates/trusted-server-core/src/integrations/registry.rs + git commit -m "Add GPT GAM attribution option" + ``` + +## Task 2: Put activation metadata on only the publisher bundle tag + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/registry.rs:1043-1054` +- Modify: `crates/trusted-server-core/src/tsjs.rs:11-39` +- Modify: `crates/trusted-server-core/src/html_processor.rs:324-360` +- Test: `crates/trusted-server-core/src/integrations/registry.rs` +- Test: `crates/trusted-server-core/src/tsjs.rs:161-187` +- Test: `crates/trusted-server-core/src/html_processor.rs:768-820` +- Test: `crates/trusted-server-core/src/html_processor.rs:1637-1671` + +- [ ] **Step 1: Write failing registry and tag-rendering tests.** + + Add a test head injector whose metadata method returns the attribution pair. + Assert registry aggregation is deterministic and preserves the default-empty + behavior of injectors that implement only `head_inserts`. + + Add exact tag tests: + + ```rust + #[test] + fn publisher_script_tag_renders_static_attributes() { + let ids = ["gpt"]; + let src = tsjs_script_src(&ids); + + assert_eq!( + tsjs_script_tag_with_attributes( + &ids, + &[("data-ts-gam-attribution", "true")] + ), + format!( + "" + ) + ); + assert_eq!( + tsjs_script_tag(&ids), + format!("") + ); + } + ``` + + The final string must contain no formatting whitespace introduced only by the + multiline example. + +- [ ] **Step 2: Write a failing HTML matrix test.** + + Process `` with: + 1. a real enabled GPT registry with attribution true; + 2. enabled GPT with attribution false; and + 3. no GPT integration. + + Assert exactly one `#trustedserver-js` tag in every case, the attribute only + in case 1, and integration head inserts remain before the external bundle. + Also retain the generic `tsjs_unified_script_tag()` exact-output test so + creative/all-modules callers stay unmarked. + +- [ ] **Step 3: Run the tests and confirm RED.** + + ```bash + cargo test-fastly tsjs_script_tag + cargo test-fastly integration_head_injector + ``` + + Expected: FAIL because the registry aggregator and attributed publisher + helper are not implemented. + +- [ ] **Step 4: Aggregate integration-owned static attributes.** + + Add beside `IntegrationRegistry::head_inserts`: + + ```rust + #[must_use] + pub fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + self.inner + .head_injectors + .iter() + .flat_map(|injector| injector.tsjs_script_tag_attributes()) + .collect() + } + ``` + + Keep the hook default-empty so existing integration injectors and test doubles + compile without changes. + +- [ ] **Step 5: Add the publisher-only tag helper.** + + Render only trusted, compile-time static attribute pairs: + + ```rust + #[must_use] + pub fn tsjs_script_tag_with_attributes( + module_ids: &[&str], + attributes: &[(&'static str, &'static str)], + ) -> String { + let attributes = attributes + .iter() + .map(|(name, value)| format!(" {name}=\"{value}\"")) + .collect::(); + format!( + "", + tsjs_script_src(module_ids) + ) + } + ``` + + Have `tsjs_script_tag(module_ids)` retain its exact output, either directly or + by delegating with an empty slice. Do not change + `tsjs_unified_script_tag()` or either creative call site. + +- [ ] **Step 6: Wire only the publisher HTML path.** + + Replace the single `html_processor.rs` call with: + + ```rust + let immediate_ids = integrations.js_module_ids_immediate(); + let script_attributes = integrations.tsjs_script_tag_attributes(); + snippet.push_str(&tsjs::tsjs_script_tag_with_attributes( + &immediate_ids, + &script_attributes, + )); + ``` + + Preserve source order: ad slots, integration head inserts, diagnostics + bootstrap, one synchronous bundle, diagnostics module, deferred bundles. + +- [ ] **Step 7: Run focused tests.** + + ```bash + cargo test-fastly tsjs_script_tag + cargo test-fastly integration_head_injector + cargo test-fastly golden_script_tag + ``` + + Expected: PASS; false/non-GPT/generic output is unmarked and true output has + one attributed publisher tag. + +- [ ] **Step 8: Commit.** + + ```bash + git add crates/trusted-server-core/src/integrations/registry.rs crates/trusted-server-core/src/tsjs.rs crates/trusted-server-core/src/html_processor.rs + git commit -m "Authorize GAM attribution bundle" + ``` + +## Task 3: Queue the primary marker before the bootstrap guard + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/gpt_bootstrap.js:17-45` +- Modify: `crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts:8-220` +- Test: `crates/trusted-server-core/src/integrations/gpt.rs:1131-1454` + +- [ ] **Step 1: Extend the raw-source test harness.** + + Add the optional page flag and `setConfig` surface: + + ```typescript + interface MockGoogleTag { + cmd: MockCommandQueue + setConfig?: (config: Record) => void + // retain the existing members + } + + type TestWindow = Omit & { + googletag?: MockGoogleTag + tsjs?: Partial + __tsjs_gam_attribution_enabled?: boolean + } + + function makeGoogleTag( + overrides: Partial = {} + ): MockGoogleTag { + return { + cmd: [], + defineSlot: vi.fn(), + pubads: vi.fn(() => ({})), + enableServices: vi.fn(), + display: vi.fn(), + ...overrides, + } + } + ``` + + Delete the flag in both `beforeEach` and `afterEach`. + +- [ ] **Step 2: Write failing behavioral tests.** + + Cover all of these independently: + - default/false plus a preinstalled `ts.adInit` returns without creating + `window.googletag`; + - true queues the exact string-valued targeting callback before a publisher + callback appended after `runBootstrap()`; + - true plus preinstalled `ts.adInit` still queues and applies targeting but + does not replace `adInit` or install the fallback scheduler; + - missing `setConfig` is a no-op and the initial-load detector and `adInit` + still install; + - throwing `setConfig` is caught inside the marker callback, and a later + publisher callback still executes; + - the wrapped `disableInitialLoad` path still records + `ts.gptInitialLoadDisabled`. + + Use a real array queue, append a publisher spy after bootstrap execution, and + drain a snapshot in order: + + ```typescript + const queue: Array<() => void> = [] + const setConfig = vi.fn() + ;(window as TestWindow).googletag = makeGoogleTag({ cmd: queue, setConfig }) + ;(window as TestWindow).__tsjs_gam_attribution_enabled = true + + runBootstrap() + const publisherCommand = vi.fn() + queue.push(publisherCommand) + ;[...queue].forEach((command) => command()) + + expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }) + expect(setConfig.mock.invocationCallOrder[0]).toBeLessThan( + publisherCommand.mock.invocationCallOrder[0] + ) + ``` + +- [ ] **Step 3: Run the raw bootstrap tests and confirm RED.** + + ```bash + cd crates/trusted-server-js/lib + npx vitest run test/integrations/gpt/gpt_bootstrap.test.ts + ``` + + Expected: targeting assertions fail because the raw bootstrap returns before + any marker enqueue. + +- [ ] **Step 4: Implement one flag-gated queue initialization before the guard.** + + Preserve the local `ts` namespace and reuse `tag` in the existing detector: + + ```javascript + var ts = (window.tsjs = window.tsjs || {}) + var tag + + if (window.__tsjs_gam_attribution_enabled === true) { + tag = window.googletag = window.googletag || { cmd: [] } + tag.cmd = tag.cmd || [] + tag.cmd.push(function () { + try { + var gpt = window.googletag + if (gpt && typeof gpt.setConfig === 'function') { + // "ts" is the fixed GAM key, not the local window.tsjs alias. + gpt.setConfig({ targeting: { ts: 'true' } }) + } + } catch (_) { + // Attribution must not interrupt the existing bootstrap queue. + } + }) + } + + if (ts.adInit) return + + tag = tag || (window.googletag = window.googletag || { cmd: [] }) + tag.cmd = tag.cmd || [] + tag.cmd.push(function () { + // existing initial-load detector body, unchanged + }) + ``` + + Do not add a global deduplication state machine, network call, beacon, cookie + read, slot-level key, or third head insert. + +- [ ] **Step 5: Add/retain Rust source-order assertions.** + + In `gpt.rs`, assert the embedded bootstrap's attribution enqueue occurs before + `if (ts.adInit) return;` and before the executable + `googletag.display(` and `googletag.pubads().refresh(` tokens. Do not compare + against comment-only `display()`/`refresh()` text. Retain `ts_initial` + assertions and the two-insert count. + +- [ ] **Step 6: Run focused JS and Rust tests.** + + ```bash + cd crates/trusted-server-js/lib + npx vitest run test/integrations/gpt/gpt_bootstrap.test.ts + cd ../../.. + cargo test-fastly head_inserts + ``` + + Expected: PASS; default behavior is unchanged and every failure mode is + isolated from the existing bootstrap. + +- [ ] **Step 7: Commit.** + + ```bash + git add crates/trusted-server-core/src/integrations/gpt_bootstrap.js crates/trusted-server-core/src/integrations/gpt.rs crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts + git commit -m "Queue page-level GAM attribution" + ``` + +## Task 4: Add the exact-executing-tag bundle fallback + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts:259-307` +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts:1878-1898` +- Modify: `crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts:350-421` + +- [ ] **Step 1: Add a test helper that controls `document.currentScript`.** + + In the runtime-gating suite, install a configurable getter before a fresh + dynamic import and restore it afterward: + + ```typescript + let executingScript: HTMLScriptElement | null + + Object.defineProperty(document, 'currentScript', { + configurable: true, + get: () => executingScript, + }) + ``` + + Use actual `" - .to_string(), + format!( + "" + ), format!("", GPT_BOOTSTRAP_JS), ]; @@ -508,6 +519,14 @@ impl IntegrationHeadInjector for GptIntegration { scripts } + + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + if self.config.gam_attribution_enabled { + vec![("data-ts-gam-attribution", "true")] + } else { + Vec::new() + } + } } /// Inline `window.tsjs.adInit` bootstrap injected at `` so the bids @@ -549,6 +568,7 @@ mod tests { fn test_config() -> GptConfig { GptConfig { enabled: true, + gam_attribution_enabled: false, script_url: default_script_url(), cache_ttl_seconds: 3600, rewrite_script: true, @@ -573,6 +593,29 @@ mod tests { .expect("should build HTTP request") } + #[test] + fn gam_attribution_defaults_to_disabled() { + let config: GptConfig = + serde_json::from_value(serde_json::json!({})).expect("should parse defaults"); + + assert!(!config.gam_attribution_enabled); + } + + #[test] + fn gam_attribution_deserializes_explicit_values() { + let disabled: GptConfig = serde_json::from_value(serde_json::json!({ + "gam_attribution_enabled": false + })) + .expect("should parse explicit false"); + let enabled: GptConfig = serde_json::from_value(serde_json::json!({ + "gam_attribution_enabled": true + })) + .expect("should parse explicit true"); + + assert!(!disabled.gam_attribution_enabled); + assert!(enabled.gam_attribution_enabled); + } + // -- URL detection -- #[test] @@ -1146,6 +1189,38 @@ mod tests { "", "should set the enable flag and call the GPT shim activation function" ); + assert!( + integration.tsjs_script_tag_attributes().is_empty(), + "should not authorize GAM attribution metadata by default" + ); + } + + #[test] + fn gam_attribution_true_adds_both_activation_signals_without_a_new_insert() { + let integration = GptIntegration::new(GptConfig { + gam_attribution_enabled: true, + ..test_config() + }); + let document_state = IntegrationDocumentState::default(); + let context = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + document_state: &document_state, + }; + + let inserts = integration.head_inserts(&context); + + assert_eq!(inserts.len(), 2, "should not add another head insert"); + assert!( + inserts[0].contains("window.__tsjs_gam_attribution_enabled=true;"), + "should activate the early bootstrap marker" + ); + assert_eq!( + integration.tsjs_script_tag_attributes(), + vec![("data-ts-gam-attribution", "true")], + "should authorize the bundle fallback on the publisher tag" + ); } #[test] diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 16cbac868..66d56e4dc 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -575,6 +575,11 @@ pub trait IntegrationHeadInjector: Send + Sync { fn integration_id(&self) -> &'static str; /// Return HTML snippets to insert at the start of ``. fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec; + + /// Return attributes to add to the publisher TSJS bundle tag. + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + Vec::new() + } } /// Registration payload returned by integration builders. From 990b6bfdb866770d98732b0fab945ea848c9f67e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 14 Aug 2026 20:22:34 +0530 Subject: [PATCH 242/395] Authorize GAM attribution bundle --- .../trusted-server-core/src/html_processor.rs | 71 ++++++++++++++++++- .../src/integrations/registry.rs | 62 ++++++++++++++++ crates/trusted-server-core/src/tsjs.rs | 44 ++++++++++-- 3 files changed, 171 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 3bff588fe..12d5b3636 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -356,7 +356,11 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso } // Main bundle: core + non-deferred integrations (synchronous). let immediate_ids = integrations.js_module_ids_immediate(); - snippet.push_str(&tsjs::tsjs_script_tag(&immediate_ids)); + let script_attributes = integrations.tsjs_script_tag_attributes(); + snippet.push_str(&tsjs::tsjs_script_tag_with_attributes( + &immediate_ids, + &script_attributes, + )); // Active diagnostics loads synchronously after core so its // GPT listeners precede publisher scripts in the origin head. if let Some(module_tag) = gpt_diagnostics @@ -835,6 +839,71 @@ mod tests { ); } + #[test] + fn integration_head_injector_marks_only_attribution_enabled_gpt_bundle() { + fn process(gam_attribution_enabled: Option) -> String { + let integrations = if let Some(gam_attribution_enabled) = gam_attribution_enabled { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "gpt", + &json!({ + "enabled": true, + "gam_attribution_enabled": gam_attribution_enabled + }), + ) + .expect("should insert GPT config"); + IntegrationRegistry::new(&settings).expect("should build GPT registry") + } else { + IntegrationRegistry::empty_for_tests() + }; + let mut config = create_test_config(); + config.integrations = integrations; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"", true) + .expect("should process HTML"); + + String::from_utf8(output).expect("should produce valid UTF-8") + } + + let attributed = process(Some(true)); + let unattributed = process(Some(false)); + let without_gpt = process(None); + + for html in [&attributed, &unattributed, &without_gpt] { + assert_eq!( + html.matches("id=\"trustedserver-js\"").count(), + 1, + "should emit exactly one publisher bundle tag: {html}" + ); + } + assert!( + attributed.contains("data-ts-gam-attribution=\"true\""), + "should mark only an attribution-enabled GPT publisher bundle" + ); + assert!( + !unattributed.contains("data-ts-gam-attribution"), + "should leave an attribution-disabled GPT publisher bundle unmarked" + ); + assert!( + !without_gpt.contains("data-ts-gam-attribution"), + "should leave a non-GPT publisher bundle unmarked" + ); + + let head_insert_index = attributed + .find("window.__tsjs_installGptShim") + .expect("should include the GPT head insert"); + let publisher_bundle_index = attributed + .find("id=\"trustedserver-js\"") + .expect("should include the publisher bundle"); + assert!( + head_insert_index < publisher_bundle_index, + "should keep integration head inserts before the publisher bundle" + ); + } + #[test] fn active_gpt_diagnostics_loads_standalone_after_unified_bundle_once() { let html = "Test"; diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 66d56e4dc..fb93ac5b3 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1058,6 +1058,16 @@ impl IntegrationRegistry { inserts } + /// Collect static attributes for the publisher TSJS bundle tag. + #[must_use] + pub fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + self.inner + .head_injectors + .iter() + .flat_map(|injector| injector.tsjs_script_tag_attributes()) + .collect() + } + /// Provide a snapshot of registered integrations and their hooks. #[must_use] pub fn registered_integrations(&self) -> Vec { @@ -1326,6 +1336,58 @@ mod tests { use crate::platform::test_support::noop_services; use http::{HeaderValue, StatusCode, header}; + struct DefaultMetadataHeadInjector; + + impl IntegrationHeadInjector for DefaultMetadataHeadInjector { + fn integration_id(&self) -> &'static str { + "default-metadata" + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + Vec::new() + } + } + + struct StaticMetadataHeadInjector; + + impl IntegrationHeadInjector for StaticMetadataHeadInjector { + fn integration_id(&self) -> &'static str { + "static-metadata" + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + Vec::new() + } + + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + vec![ + ("data-ts-gam-attribution", "true"), + ("data-test-order", "second"), + ] + } + } + + #[test] + fn tsjs_script_tag_attributes_preserve_registration_order_and_default_empty() { + let registry = IntegrationRegistry::from_rewriters_with_head_injectors( + Vec::new(), + Vec::new(), + vec![ + Arc::new(DefaultMetadataHeadInjector), + Arc::new(StaticMetadataHeadInjector), + ], + ); + + assert_eq!( + registry.tsjs_script_tag_attributes(), + vec![ + ("data-ts-gam-attribution", "true"), + ("data-test-order", "second"), + ], + "should omit default-empty metadata and preserve registered attribute order" + ); + } + // Mock integration proxy for testing struct MockProxy; diff --git a/crates/trusted-server-core/src/tsjs.rs b/crates/trusted-server-core/src/tsjs.rs index 133e6d011..f4c9a13a5 100644 --- a/crates/trusted-server-core/src/tsjs.rs +++ b/crates/trusted-server-core/src/tsjs.rs @@ -11,9 +11,23 @@ pub fn tsjs_script_src(module_ids: &[&str]) -> String { /// `", - tsjs_script_src(module_ids) + "", + tsjs_script_src(module_ids), ) } @@ -170,19 +184,39 @@ mod tests { ); } + #[test] + fn publisher_tsjs_script_tag_renders_static_attributes() { + let module_ids = ["gpt"]; + let src = tsjs_script_src(&module_ids); + + assert_eq!( + tsjs_script_tag_with_attributes(&module_ids, &[("data-ts-gam-attribution", "true")]), + format!( + "" + ), + "should render trusted static attributes on the publisher bundle tag" + ); + assert_eq!( + tsjs_script_tag(&module_ids), + format!(""), + "should keep the generic tag byte-for-byte unmarked" + ); + } + #[test] fn tsjs_unified_helpers_use_all_module_ids() { let ids = all_module_ids(); + let src = tsjs_unified_script_src(); assert_eq!( - tsjs_unified_script_src(), + src, tsjs_script_src(&ids), "should hash all module IDs for the unified script source" ); assert_eq!( tsjs_unified_script_tag(), - tsjs_script_tag(&ids), - "should wrap the all-module unified script source" + format!(""), + "should keep the all-module generic tag byte-for-byte unmarked" ); } From 0bfee82a58863c5c11268ff3069ae307b3fc23c9 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 14 Aug 2026 20:26:23 +0530 Subject: [PATCH 243/395] Cover disabled GPT attribution --- .../trusted-server-core/src/html_processor.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 12d5b3636..10dce6658 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -841,15 +841,15 @@ mod tests { #[test] fn integration_head_injector_marks_only_attribution_enabled_gpt_bundle() { - fn process(gam_attribution_enabled: Option) -> String { - let integrations = if let Some(gam_attribution_enabled) = gam_attribution_enabled { + fn process(gpt_config: Option<(bool, bool)>) -> String { + let integrations = if let Some((enabled, gam_attribution_enabled)) = gpt_config { let mut settings = create_test_settings(); settings .integrations .insert_config( "gpt", &json!({ - "enabled": true, + "enabled": enabled, "gam_attribution_enabled": gam_attribution_enabled }), ) @@ -868,11 +868,12 @@ mod tests { String::from_utf8(output).expect("should produce valid UTF-8") } - let attributed = process(Some(true)); - let unattributed = process(Some(false)); + let attributed = process(Some((true, true))); + let unattributed = process(Some((true, false))); + let disabled_gpt = process(Some((false, true))); let without_gpt = process(None); - for html in [&attributed, &unattributed, &without_gpt] { + for html in [&attributed, &unattributed, &disabled_gpt, &without_gpt] { assert_eq!( html.matches("id=\"trustedserver-js\"").count(), 1, @@ -887,6 +888,10 @@ mod tests { !unattributed.contains("data-ts-gam-attribution"), "should leave an attribution-disabled GPT publisher bundle unmarked" ); + assert!( + !disabled_gpt.contains("data-ts-gam-attribution"), + "should let the GPT master switch suppress attribution metadata" + ); assert!( !without_gpt.contains("data-ts-gam-attribution"), "should leave a non-GPT publisher bundle unmarked" From 06ec21326e7f2cc88cbfc637e197ebaffd89e358 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 14 Aug 2026 20:31:13 +0530 Subject: [PATCH 244/395] Queue page-level GAM attribution --- .../src/integrations/gpt.rs | 29 ++++ .../src/integrations/gpt_bootstrap.js | 22 +++- .../integrations/gpt/gpt_bootstrap.test.ts | 124 ++++++++++++++++++ 3 files changed, 174 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index f4f09bbff..533944027 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -1427,6 +1427,35 @@ mod tests { ); } + #[test] + fn head_inserts_queue_gam_attribution_before_guard_and_ad_requests() { + let targeting_index = GPT_BOOTSTRAP_JS + .find("gpt.setConfig({ targeting: { ts: 'true' } })") + .expect("should apply the fixed page-level GAM targeting pair"); + let guard_index = GPT_BOOTSTRAP_JS + .find("if (ts.adInit) return;") + .expect("should retain the preinstalled adInit guard"); + let display_index = GPT_BOOTSTRAP_JS + .find("googletag.display(divId);") + .expect("should retain the executable GPT display call"); + let refresh_index = GPT_BOOTSTRAP_JS + .find("googletag.pubads().refresh(slotsNeedingRefresh);") + .expect("should retain the bounded GPT refresh call"); + + assert!( + targeting_index < guard_index, + "should enqueue attribution before the preinstalled adInit guard" + ); + assert!( + targeting_index < display_index, + "should enqueue attribution before the executable display call" + ); + assert!( + targeting_index < refresh_index, + "should enqueue attribution before the executable refresh call" + ); + } + #[test] fn head_injector_integration_id() { let integration = GptIntegration::new(test_config()); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 86b51ffa7..e336d0438 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -17,6 +17,24 @@ (function () { if (typeof window === "undefined") return; var ts = (window.tsjs = window.tsjs || {}); + var tag; + + if (window.__tsjs_gam_attribution_enabled === true) { + tag = window.googletag = window.googletag || { cmd: [] }; + tag.cmd = tag.cmd || []; + tag.cmd.push(function () { + try { + var gpt = window.googletag; + if (gpt && typeof gpt.setConfig === "function") { + // "ts" is the fixed GAM key, not the local window.tsjs alias. + gpt.setConfig({ targeting: { ts: 'true' } }); + } + } catch (_) { + // Attribution must not interrupt the existing bootstrap queue. + } + }); + } + if (ts.adInit) return; // Track whether the publisher disabled GPT initial load. Read the effective @@ -38,7 +56,9 @@ return true; } - (window.googletag = window.googletag || { cmd: [] }).cmd.push(function () { + tag = tag || (window.googletag = window.googletag || { cmd: [] }); + tag.cmd = tag.cmd || []; + tag.cmd.push(function () { var gpt = window.googletag; syncInitialLoadDisabled(gpt); if ( diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index d3e1d7099..4ec4110b5 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -33,6 +33,8 @@ interface MockGoogleTag { pubads: () => unknown; enableServices: () => void; display: (divId: string) => void; + getConfig?: (key: string) => Record; + setConfig?: (config: Record) => void; } // `tsjs` is declared globally as the full `TsjsApi`; `Omit` drops it from @@ -40,8 +42,25 @@ interface MockGoogleTag { type TestWindow = Omit & { googletag?: MockGoogleTag; tsjs?: Partial; + __tsjs_gam_attribution_enabled?: boolean; }; +function makeGoogleTag(overrides: Partial = {}): MockGoogleTag { + const pubads = { + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + + return { + cmd: [], + defineSlot: vi.fn(), + pubads: vi.fn(() => pubads), + enableServices: vi.fn(), + display: vi.fn(), + ...overrides, + }; +} + function runBootstrap(): void { // Evaluate in the jsdom global scope, exactly as an inline -"#; +const APS_RENDERER_DOCUMENT: &str = + include_str!("../../../trusted-server-js/lib/src/integrations/aps/renderer.html"); +const APS_RENDERER_BOOTSTRAP_DOCUMENT: &str = + include_str!("../../../trusted-server-js/lib/src/integrations/aps/renderer-bootstrap.html"); /// Configuration for the APS `OpenRTB` integration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] @@ -1211,13 +1152,28 @@ impl IntegrationProxy for ApsRendererIntegration { message: "Failed to build APS not-found response".to_string(), }); } + let (renderer_document, renderer_csp) = match request.uri().query() { + None => (APS_RENDERER_DOCUMENT, APS_RENDERER_CSP), + Some(APS_RENDERER_BOOTSTRAP_QUERY) => { + (APS_RENDERER_BOOTSTRAP_DOCUMENT, APS_RENDERER_BOOTSTRAP_CSP) + } + Some(_) => { + return http::Response::builder() + .status(StatusCode::NOT_FOUND) + .body(EdgeBody::from("Not Found")) + .change_context(TrustedServerError::Integration { + integration: APS_INTEGRATION_ID.to_string(), + message: "Failed to build APS not-found response".to_string(), + }); + } + }; http::Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "text/html; charset=utf-8") .header("x-content-type-options", "nosniff") .header("referrer-policy", "no-referrer") - .header(header::CONTENT_SECURITY_POLICY, APS_RENDERER_CSP) - .body(EdgeBody::from(APS_RENDERER_DOCUMENT)) + .header(header::CONTENT_SECURITY_POLICY, renderer_csp) + .body(EdgeBody::from(renderer_document)) .change_context(TrustedServerError::Integration { integration: APS_INTEGRATION_ID.to_string(), message: "Failed to build APS renderer response".to_string(), @@ -2318,7 +2274,7 @@ mod tests { } #[test] - fn registers_and_serves_only_static_renderer_route() { + fn registers_and_serves_static_renderer_and_data_bootstrap_modes() { let integration = ApsRendererIntegration; let routes = integration.routes(); assert_eq!(routes.len(), 1, "should register one route"); @@ -2347,6 +2303,25 @@ mod tests { APS_RENDERER_CSP ); + let bootstrap = http::Request::builder() + .method(Method::GET) + .uri(format!( + "{APS_RENDERER_ROUTE}?{APS_RENDERER_BOOTSTRAP_QUERY}" + )) + .body(EdgeBody::empty()) + .expect("should build renderer bootstrap request"); + let response = + futures::executor::block_on(integration.handle(&settings, &services, bootstrap)) + .expect("should serve renderer bootstrap"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_SECURITY_POLICY], + APS_RENDERER_BOOTSTRAP_CSP + ); + assert!(APS_RENDERER_BOOTSTRAP_CSP.contains("allow-same-origin")); + assert!(APS_RENDERER_BOOTSTRAP_CSP.contains("frame-src https: data:")); + assert!(APS_RENDERER_BOOTSTRAP_DOCUMENT.contains("trusted-server/aps/bootstrap-navigate")); + let post = http::Request::builder() .method(Method::POST) .uri(APS_RENDERER_ROUTE) @@ -2374,6 +2349,7 @@ mod tests { assert_eq!(registration.integration_id, APS_INTEGRATION_ID); assert_eq!(registration.proxies.len(), 1); + assert!(registration.request_filters.is_empty()); assert!(registration.js_disabled); } @@ -2425,12 +2401,15 @@ mod tests { #[test] fn renderer_document_is_static_and_nonce_bound() { assert!(APS_RENDERER_DOCUMENT.contains("^#tsaps=")); - assert!(APS_RENDERER_DOCUMENT.contains("event.source!==parent")); - assert!(APS_RENDERER_DOCUMENT.contains("message.nonce!==expected")); + assert!(APS_RENDERER_DOCUMENT.contains("event.source !== parent")); + assert!(APS_RENDERER_DOCUMENT.contains("message.nonce !== expected")); + assert!(APS_RENDERER_DOCUMENT.contains("['nonce', 'publisherOrigin', 'renderer']")); + assert!(APS_RENDERER_DOCUMENT.contains("['nonce', 'renderer']")); + assert!(APS_RENDERER_DOCUMENT.contains("url.origin === publisherOrigin")); assert!(APS_RENDERER_DOCUMENT.contains("prebid/creative/render")); assert!(APS_RENDERER_DOCUMENT.contains("window._aps instanceof Map")); - assert!(APS_RENDERER_DOCUMENT.contains("store:new Map([['listeners',new Map()]])")); - assert!(APS_RENDERER_DOCUMENT.contains("account.queue.push(new CustomEvent")); + assert!(APS_RENDERER_DOCUMENT.contains("store: new Map([['listeners', new Map()]])")); + assert!(APS_RENDERER_DOCUMENT.contains("account.queue.push(")); assert!( APS_RENDERER_DOCUMENT.contains("trusted-server/aps/renderer-ready") && APS_RENDERER_DOCUMENT.contains("trusted-server/aps/renderer-failed") @@ -2442,7 +2421,7 @@ mod tests { ); assert!(!APS_RENDERER_DOCUMENT.contains(" diff --git a/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html b/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html new file mode 100644 index 000000000..6ef871c9b --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html @@ -0,0 +1,71 @@ + + + + + + + + diff --git a/crates/trusted-server-js/lib/src/integrations/aps/renderer.html b/crates/trusted-server-js/lib/src/integrations/aps/renderer.html new file mode 100644 index 000000000..b7ddac1c0 --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/aps/renderer.html @@ -0,0 +1,228 @@ + + + + + diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index f0df35974..e861f8b41 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -10,9 +10,9 @@ import type { import { APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - apsRendererUrl, consumeApsPrebidRenderer, getApsPrebidRenderer, + registerApsUniversalCreativeMount, validateApsRenderer, } from '../aps/render'; @@ -225,13 +225,14 @@ function slotIdForMessageSource(source: MessageEventSource | null): string | und ?.id; } -function messageSourceBelongsToAdUnit( +function slotRootForMessageSource( source: MessageEventSource | null, - adUnitCode: string -): boolean { - return source - ? sourceIsInSlotRoots(source, candidateSlotRootsForConfiguredDivId(adUnitCode)) - : false; + divId: string +): HTMLElement | undefined { + if (!source) return undefined; + return candidateSlotRootsForConfiguredDivId(divId).find((root) => + sourceIsInSlotRoots(source, [root]) + ); } function clearTargetingKeys(slot: GoogleTagSlot, keys: Iterable): void { @@ -1674,13 +1675,15 @@ export function installTsRenderBridge(): void { // Prebid handles ad IDs globally and would otherwise answer a request from // an unrelated iframe when this slot-bound capability rejects it. e.stopImmediatePropagation(); - if (!messageSourceBelongsToAdUnit(e.source, prebidRendererEntry.adUnitCode)) return; + const mountContainer = slotRootForMessageSource(e.source, prebidRendererEntry.adUnitCode); + if (!mountContainer) return; const renderer = validateApsRenderer(prebidRendererEntry.renderer); - const rendererUrl = apsRendererUrl(); - if (!renderer || !rendererUrl) return; + if (!renderer) return; if (!hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; if (!consumeApsPrebidRenderer(adId, prebidRendererEntry)) return; recordConsumedPrebidApsId(consumedPrebidApsIds, adId, prebidRendererEntry.expiresAt); + const apsMountId = registerApsUniversalCreativeMount(mountContainer, renderer); + if (!apsMountId) return; port.postMessage( JSON.stringify({ @@ -1688,7 +1691,8 @@ export function installTsRenderBridge(): void { adId, renderer: APS_UNIVERSAL_CREATIVE_RENDERER, rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - rendererUrl, + apsMountId, + publisherOrigin: window.location.origin, apsRenderer: renderer, width: renderer.width, height: renderer.height, @@ -1727,8 +1731,11 @@ export function installTsRenderBridge(): void { e.stopImmediatePropagation(); if (consumedServerApsBySlot.get(slotId) === adId) return; const renderer = validateApsRenderer(matchedBid.renderer); - const rendererUrl = apsRendererUrl(); - if (!renderer || !rendererUrl) return; + const configuredSlot = window.tsjs?.adSlots?.find((slot) => slot.id === slotId); + const mountContainer = slotRootForMessageSource(e.source, configuredSlot?.div_id ?? slotId); + if (!renderer || !mountContainer) return; + const apsMountId = registerApsUniversalCreativeMount(mountContainer, renderer); + if (!apsMountId) return; consumedServerApsBySlot.set(slotId, adId); port.postMessage( JSON.stringify({ @@ -1736,7 +1743,8 @@ export function installTsRenderBridge(): void { adId, renderer: APS_UNIVERSAL_CREATIVE_RENDERER, rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - rendererUrl, + apsMountId, + publisherOrigin: window.location.origin, apsRenderer: renderer, width: renderer.width, height: renderer.height, diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index dc17c9e87..ba2dfb274 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -69,7 +69,7 @@ describe('request.requestAds', () => { ); }); - it('dispatches a valid APS descriptor to the opaque static renderer route', async () => { + it('dispatches a valid APS descriptor through the opaque data renderer bootstrap', async () => { const apsBid = envelope.seatbid[0].bid[0]; const renderer = { type: 'aps', @@ -117,21 +117,55 @@ describe('request.requestAds', () => { const iframe = document.querySelector('#slot1 iframe') as HTMLIFrameElement | null; expect(iframe).not.toBeNull(); - expect(iframe!.src).toContain('/integrations/aps/renderer#tsaps='); + expect(iframe!.src).toContain('/integrations/aps/renderer?mode=data-bootstrap#tsaps='); expect(iframe!.srcdoc).toBe(''); expect(iframe!.getAttribute('sandbox')).not.toContain('allow-same-origin'); expect(document.querySelector('#slot1 span')).not.toBeNull(); const postMessage = vi.spyOn(iframe!.contentWindow!, 'postMessage'); - iframe!.dispatchEvent(new Event('load')); + const nonce = new URL(iframe!.src).hash.replace('#tsaps=', ''); + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/bootstrap-ready', nonce }, + source: iframe!.contentWindow, + }) + ); + const navigate = postMessage.mock.calls[0][0] as { rendererUrl: string }; + const containerDocument = decodeURIComponent( + navigate.rendererUrl + .slice('data:text/html;charset=utf-8,'.length) + .replace(/#tsaps=[A-Za-z0-9_-]{22}$/, '') + ); + const innerNonce = containerDocument.match( + /data:text\/html;charset=utf-8,.+#tsaps=([A-Za-z0-9_-]{22})/ + )?.[1]; + expect(innerNonce).toMatch(/^[A-Za-z0-9_-]{22}$/); + + const channel = { + close: vi.fn(), + onmessage: null, + postMessage: vi.fn(), + start: vi.fn(), + } as unknown as MessagePort; + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/container-ready', nonce }, + source: iframe!.contentWindow, + ports: [channel], + }) + ); + channel.onmessage?.( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/channel-ready', nonce: innerNonce }, + }) + ); expect(document.querySelector('#slot1 span')).not.toBeNull(); - expect(postMessage).toHaveBeenCalledWith(expect.objectContaining({ renderer }), '*'); + expect(channel.postMessage).toHaveBeenCalledWith(expect.objectContaining({ renderer })); - const message = postMessage.mock.calls[0][0] as { nonce: string }; - window.dispatchEvent( + const message = vi.mocked(channel.postMessage).mock.calls[0][0] as { nonce: string }; + channel.onmessage?.( new MessageEvent('message', { data: { message: 'trusted-server/aps/renderer-ready', nonce: message.nonce }, - source: iframe!.contentWindow, }) ); expect(document.querySelector('#slot1 span')).toBeNull(); diff --git a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts index eae60c90e..c9808b8ac 100644 --- a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts @@ -4,14 +4,18 @@ import envelope from '../../fixtures/aps-renderer-v1.json'; import type { ApsRendererV1 } from '../../../src/core/types'; import { log } from '../../../src/core/log'; import { + APS_RENDERER_DATA_URL, APS_RENDERER_PATH, APS_RENDERER_SANDBOX, APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, + apsRendererBootstrapUrl, apsRendererUrl, + cancelPendingApsRender, getApsPrebidRenderer, parseApsRendererDescriptor, registerApsPrebidRenderer, + registerApsUniversalCreativeMount, renderApsCreative, validateApsRenderer, } from '../../../src/integrations/aps/render'; @@ -50,6 +54,76 @@ function descriptor(overrides: Partial = {}): ApsRendererV1 { }; } +type FakeRendererChannel = MessagePort & { + close: ReturnType; + postMessage: ReturnType; + start: ReturnType; +}; + +function sendRendererMessage(channel: FakeRendererChannel, data: Record): void { + channel.onmessage?.(new MessageEvent('message', { data })); +} + +function advanceRendererToData(iframe: HTMLIFrameElement) { + const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); + const nonce = new URL(iframe.src).hash.replace('#tsaps=', ''); + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/bootstrap-ready', nonce }, + source: iframe.contentWindow, + }) + ); + expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); + const navigate = postMessage.mock.calls[0][0] as { + message: string; + nonce: string; + rendererUrl: string; + }; + expect(navigate.message).toBe('trusted-server/aps/bootstrap-navigate'); + expect(navigate.nonce).toBe(nonce); + expect(navigate.rendererUrl).toMatch( + /^data:text\/html;charset=utf-8,.+#tsaps=[A-Za-z0-9_-]{22}$/ + ); + + const encodedDocument = navigate.rendererUrl + .slice('data:text/html;charset=utf-8,'.length) + .replace(/#tsaps=[A-Za-z0-9_-]{22}$/, ''); + const containerDocument = decodeURIComponent(encodedDocument); + const innerNonce = containerDocument.match( + /data:text\/html;charset=utf-8,.+#tsaps=([A-Za-z0-9_-]{22})/ + )?.[1]; + expect(innerNonce).toMatch(/^[A-Za-z0-9_-]{22}$/); + expect(innerNonce).not.toBe(nonce); + expect(containerDocument).toContain('frame-src data: https://creative.example'); + expect(containerDocument).not.toContain(descriptor().bidId); + expect(containerDocument).not.toContain(descriptor().aaxResponse); + + const channel = { + close: vi.fn(), + onmessage: null, + postMessage: vi.fn(), + start: vi.fn(), + } as unknown as FakeRendererChannel; + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/container-ready', nonce }, + source: iframe.contentWindow, + ports: [channel], + }) + ); + expect(channel.start).toHaveBeenCalledOnce(); + sendRendererMessage(channel, { + message: 'trusted-server/aps/channel-ready', + nonce: innerNonce, + }); + const sent = channel.postMessage.mock.calls[0][0] as { + nonce: string; + publisherOrigin: string; + renderer: ApsRendererV1; + }; + return { channel, innerNonce: innerNonce!, postMessage, sent }; +} + describe('APS renderer validation', () => { it('consumes the shared fictional golden envelope and supports an omitted creative ID', () => { const withCreativeId = descriptor(); @@ -271,62 +345,58 @@ describe('direct APS rendering', () => { document.body.innerHTML = ''; }); - it('loads the static route with a fragment-bound 128-bit nonce and opaque sandbox', () => { + it('bootstraps the data renderer with a fragment-bound 128-bit nonce', () => { expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const slot = document.getElementById('fictional-slot')!; const iframe = slot.querySelector('iframe')!; const existing = slot.querySelector('span'); expect(existing).not.toBeNull(); - expect(iframe.src).toMatch(/\/integrations\/aps\/renderer#tsaps=[A-Za-z0-9_-]{22}$/); - expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); + expect(iframe.src.startsWith(`${apsRendererBootstrapUrl()}#tsaps=`)).toBe(true); + expect(iframe.src).toMatch(/\?mode=data-bootstrap#tsaps=[A-Za-z0-9_-]{22}$/); expect(iframe.getAttribute('sandbox')).not.toContain('allow-same-origin'); expect(iframe.srcdoc).toBe(''); - const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); - iframe.dispatchEvent(new Event('load')); - + const { channel, sent } = advanceRendererToData(iframe); expect(slot.querySelector('span')).not.toBeNull(); expect(iframe.style.display).toBe('none'); - expect(postMessage).toHaveBeenCalledTimes(1); - expect(postMessage).toHaveBeenCalledWith( - { - nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), - renderer: descriptor(), - }, - '*' - ); + expect(iframe.getAttribute('sandbox')).toContain('allow-same-origin'); + expect(sent).toEqual({ + nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), + publisherOrigin: window.location.origin, + renderer: descriptor(), + }); + + sendRendererMessage(channel, { + message: 'trusted-server/aps/renderer-ready', + nonce: `wrong-${sent.nonce}`, + }); + expect(slot.querySelector('span')).not.toBeNull(); - const message = postMessage.mock.calls[0][0] as { nonce: string }; window.dispatchEvent( new MessageEvent('message', { - data: { - message: 'trusted-server/aps/renderer-ready', - nonce: `wrong-${message.nonce}`, - }, + data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, source: iframe.contentWindow, }) ); expect(slot.querySelector('span')).not.toBeNull(); + expect(iframe.style.display).toBe('none'); - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: message.nonce }, - source: iframe.contentWindow, - }) - ); + sendRendererMessage(channel, { + message: 'trusted-server/aps/renderer-ready', + nonce: sent.nonce, + }); + expect(iframe.getAttribute('sandbox')).toContain('allow-same-origin'); expect(slot.querySelector('span')).toBeNull(); expect(iframe.style.display).toBe(''); }); - it('rejects a ready message with the correct nonce from a foreign window', () => { + it('accepts readiness only through the transferred renderer channel', () => { expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const slot = document.getElementById('fictional-slot')!; const rendererFrame = slot.querySelector('iframe')!; - const postMessage = vi.spyOn(rendererFrame.contentWindow!, 'postMessage'); - rendererFrame.dispatchEvent(new Event('load')); - const sent = postMessage.mock.calls[0][0] as { nonce: string }; + const { channel, sent } = advanceRendererToData(rendererFrame); const foreignFrame = document.createElement('iframe'); document.body.appendChild(foreignFrame); @@ -336,10 +406,6 @@ describe('direct APS rendering', () => { source: foreignFrame.contentWindow, }) ); - - expect(slot.querySelector('span')).not.toBeNull(); - expect(rendererFrame.style.display).toBe('none'); - window.dispatchEvent( new MessageEvent('message', { data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, @@ -347,6 +413,13 @@ describe('direct APS rendering', () => { }) ); + expect(slot.querySelector('span')).not.toBeNull(); + expect(rendererFrame.style.display).toBe('none'); + + sendRendererMessage(channel, { + message: 'trusted-server/aps/renderer-ready', + nonce: sent.nonce, + }); expect(slot.querySelector('span')).toBeNull(); expect(rendererFrame.style.display).toBe(''); }); @@ -368,6 +441,16 @@ describe('direct APS rendering', () => { expect(document.querySelector('#fictional-slot iframe')).toBeNull(); }); + it('cancels a pending frame before another renderer replaces the slot', () => { + expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); + const container = document.getElementById('fictional-slot')!; + + cancelPendingApsRender(container); + + expect(container.querySelector('span')).not.toBeNull(); + expect(container.querySelector('iframe')).toBeNull(); + }); + it('removes an unacknowledged frame without clearing publisher content', () => { vi.useFakeTimers(); try { @@ -391,9 +474,9 @@ describe('direct APS rendering', () => { const baselineTimers = vi.getTimerCount(); expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const firstFrame = document.querySelector('#fictional-slot iframe')!; - const firstPostMessage = vi.spyOn(firstFrame.contentWindow!, 'postMessage'); - firstFrame.dispatchEvent(new Event('load')); - const firstSent = firstPostMessage.mock.calls[0][0] as { nonce: string }; + const { channel: firstChannel, sent: firstSent } = advanceRendererToData( + firstFrame as HTMLIFrameElement + ); const timersAfterFirst = vi.getTimerCount(); expect(timersAfterFirst).toBeGreaterThan(baselineTimers); @@ -401,25 +484,21 @@ describe('direct APS rendering', () => { const secondFrame = document.querySelector('#fictional-slot iframe')!; expect(firstFrame.isConnected).toBe(false); expect(vi.getTimerCount()).toBe(timersAfterFirst); - const postMessage = vi.spyOn(secondFrame.contentWindow!, 'postMessage'); - secondFrame.dispatchEvent(new Event('load')); - const sent = postMessage.mock.calls[0][0] as { nonce: string }; - - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: firstSent.nonce }, - source: firstFrame.contentWindow, - }) + const { channel: secondChannel, sent } = advanceRendererToData( + secondFrame as HTMLIFrameElement ); + + sendRendererMessage(firstChannel, { + message: 'trusted-server/aps/renderer-ready', + nonce: firstSent.nonce, + }); expect(document.querySelector('#fictional-slot span')).not.toBeNull(); expect((secondFrame as HTMLIFrameElement).style.display).toBe('none'); - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, - source: secondFrame.contentWindow, - }) - ); + sendRendererMessage(secondChannel, { + message: 'trusted-server/aps/renderer-ready', + nonce: sent.nonce, + }); vi.advanceTimersByTime(10_000); expect(warnSpy).not.toHaveBeenCalled(); @@ -430,19 +509,21 @@ describe('direct APS rendering', () => { }); describe('Universal Creative APS source', () => { - it('uses the deployed dynamic renderer protocol and only creates the opaque route frame', () => { - expect(APS_UNIVERSAL_CREATIVE_RENDERER_VERSION).toBeGreaterThanOrEqual(4); + it('uses the deployed dynamic renderer protocol to request a top-page mount', () => { + expect(APS_UNIVERSAL_CREATIVE_RENDERER_VERSION).toBeGreaterThanOrEqual(6); expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('window.render=function'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.apsRenderer'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.rendererUrl'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain(APS_RENDERER_PATH); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain(APS_RENDERER_SANDBOX); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('allow-same-origin'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.apsMountId'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.publisherOrigin'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('trusted-server/aps/mount-request'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('d&&d.apsRenderer'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('d&&d.rendererUrl'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(APS_RENDERER_DATA_URL); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(APS_RENDERER_SANDBOX); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('srcdoc'); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('document.write'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('creativeUrl'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('aaxResponse'); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('example-account-id'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(descriptor().creativeUrl); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(descriptor().aaxResponse); }); it('computes an absolute renderer URL from the publisher origin', () => { @@ -453,31 +534,114 @@ describe('Universal Creative APS source', () => { expect(apsRendererUrl('not an origin')).toBeUndefined(); }); - it('creates the opaque route frame and resolves only after the bound acknowledgement', async () => { + it('consumes a top-page mount capability once and preserves the controller frame', () => { + document.body.innerHTML = + '
'; + const container = document.getElementById('fictional-puc-slot')!; + const controller = container.querySelector('.puc-controller')!; + const requester = document.createElement('iframe'); + document.body.appendChild(requester); + const resultPost = vi.spyOn(requester.contentWindow!, 'postMessage'); + const mountId = registerApsUniversalCreativeMount(container, descriptor())!; + const requestNonce = 'ZYXWVUTSRQPONMLKJIHGFE'; + + const request = () => + window.dispatchEvent( + new MessageEvent('message', { + data: { + message: 'trusted-server/aps/mount-request', + mountId, + nonce: requestNonce, + }, + source: requester.contentWindow, + }) + ); + request(); + request(); + + const rendererFrame = container.querySelector( + 'iframe[data-ts-aps-renderer="true"]' + )!; + expect(rendererFrame).not.toBeNull(); + expect(container.querySelectorAll('iframe[data-ts-aps-renderer="true"]')).toHaveLength(1); + expect(controller.isConnected).toBe(true); + + const { channel, sent } = advanceRendererToData(rendererFrame); + sendRendererMessage(channel, { + message: 'trusted-server/aps/renderer-ready', + nonce: sent.nonce, + }); + + expect(controller.isConnected).toBe(true); + expect(controller.style.display).toBe('none'); + expect(rendererFrame.style.display).toBe(''); + expect(resultPost).toHaveBeenCalledWith( + { + message: 'trusted-server/aps/mount-result', + mountId, + nonce: requestNonce, + status: 'ready', + }, + '*' + ); + document.body.innerHTML = ''; + }); + + it('revokes an older mount capability when the same container is registered again', () => { + document.body.innerHTML = '
'; + const container = document.getElementById('fictional-refresh-slot')!; + const requester = document.createElement('iframe'); + document.body.appendChild(requester); + const oldMountId = registerApsUniversalCreativeMount(container, descriptor())!; + const newMountId = registerApsUniversalCreativeMount(container, descriptor())!; + const request = (mountId: string) => + window.dispatchEvent( + new MessageEvent('message', { + data: { + message: 'trusted-server/aps/mount-request', + mountId, + nonce: 'ABCDEFGHIJKLMNOPQRSTUV', + }, + source: requester.contentWindow, + }) + ); + + request(oldMountId); + expect(container.querySelector('iframe[data-ts-aps-renderer="true"]')).toBeNull(); + request(newMountId); + const rendererFrame = container.querySelector( + 'iframe[data-ts-aps-renderer="true"]' + ); + expect(rendererFrame).not.toBeNull(); + rendererFrame!.dispatchEvent(new Event('error')); + document.body.innerHTML = ''; + }); + + it('resolves only after the top page acknowledges its one-shot mount request', async () => { const dynamicWindow = window as unknown as { - render?: (data: Record, helper: unknown, target: Window) => Promise; + render?: (data: Record) => Promise; }; + const postMessage = vi.spyOn(window.top, 'postMessage'); window.eval(APS_UNIVERSAL_CREATIVE_RENDERER); try { - const renderer = descriptor(); - const rendered = dynamicWindow.render!( - { - apsRenderer: renderer, - rendererUrl: apsRendererUrl(), - }, - undefined, - window - ); - const iframe = document.body.querySelector('iframe')!; - expect(iframe.src).toMatch(/\/integrations\/aps\/renderer#tsaps=[A-Za-z0-9_-]{22}$/); - expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); - expect(iframe.getAttribute('sandbox')).not.toContain('allow-same-origin'); - - const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); - iframe.dispatchEvent(new Event('load')); - const sent = postMessage.mock.calls[0][0] as { nonce: string; renderer: ApsRendererV1 }; - expect(sent.renderer).toEqual(renderer); + const mountId = 'ABCDEFGHIJKLMNOPQRSTUV'; + const rendered = dynamicWindow.render!({ + apsMountId: mountId, + publisherOrigin: window.location.origin, + }); + expect(document.body.querySelector('iframe')).toBeNull(); + expect(postMessage).toHaveBeenCalledTimes(1); + const sent = postMessage.mock.calls[0][0] as { + message: string; + mountId: string; + nonce: string; + }; + expect(sent).toEqual({ + message: 'trusted-server/aps/mount-request', + mountId, + nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), + }); let settled = false; void rendered.then(() => { @@ -488,12 +652,18 @@ describe('Universal Creative APS source', () => { window.dispatchEvent( new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, - source: iframe.contentWindow, + data: { + message: 'trusted-server/aps/mount-result', + mountId, + nonce: sent.nonce, + status: 'ready', + }, + source: window.top, }) ); await expect(rendered).resolves.toBeUndefined(); } finally { + postMessage.mockRestore(); delete dynamicWindow.render; document.body.innerHTML = ''; } diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 179a810d5..6d73094cf 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -3143,11 +3143,12 @@ describe('installTsRenderBridge', () => { expect(Object.keys(response).sort()).toEqual( [ 'adId', + 'apsMountId', 'apsRenderer', 'height', 'message', + 'publisherOrigin', 'renderer', - 'rendererUrl', 'rendererVersion', 'width', ].sort() @@ -3156,8 +3157,9 @@ describe('installTsRenderBridge', () => { message: 'Prebid Response', adId: renderer.bidId, renderer: expect.stringContaining('window.render=function'), - rendererVersion: 4, - rendererUrl: new URL('/integrations/aps/renderer', window.location.origin).href, + rendererVersion: 6, + apsMountId: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), + publisherOrigin: window.location.origin, apsRenderer: renderer, width: 300, height: 250, @@ -3165,35 +3167,9 @@ describe('installTsRenderBridge', () => { expect(String(response.renderer)).not.toContain(renderer.accountId); expect(String(response.renderer)).not.toContain(renderer.aaxResponse); - // Universal Creative's dynamic-renderer path evaluates the returned static - // source and calls window.render(response, helper, targetWindow). Consume - // the exact bridge response through that deployed protocol shape. - const dynamicWindow = window as unknown as { - render?: (data: Record, helper: unknown, target: Window) => Promise; - }; - window.eval(String(response.renderer)); - try { - const rendered = dynamicWindow.render!(response, undefined, window); - const outerFrame = document.querySelector( - 'iframe[src*="/integrations/aps/renderer#tsaps="]' - )!; - expect(outerFrame).not.toBeNull(); - expect(outerFrame.getAttribute('sandbox')).not.toContain('allow-same-origin'); - - const rendererPost = vi.spyOn(outerFrame.contentWindow!, 'postMessage'); - outerFrame.dispatchEvent(new Event('load')); - const sent = rendererPost.mock.calls[0][0] as { nonce: string }; - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, - source: outerFrame.contentWindow, - }) - ); - await expect(rendered).resolves.toBeUndefined(); - outerFrame.remove(); - } finally { - delete dynamicWindow.render; - } + expect(String(response.renderer)).toContain('d&&d.apsMountId'); + expect(String(response.renderer)).not.toContain(renderer.creativeUrl); + expect(String(response.renderer)).not.toContain(renderer.aaxResponse); beaconSpy.mockRestore(); }); diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index 658f5d5bc..3ab6e904b 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -150,30 +150,27 @@ Trusted Server does not insert APS creative markup into the publisher document. Seats, `impid`, markup, notifications, user-sync data, sibling bids, losing seats, and unknown fields are not exposed. The browser decodes this envelope and cross-checks the ID, dimensions, URL, and tag type before any DOM mutation or message suppression. -Both rendering paths use `GET /integrations/aps/renderer`, a static Trusted Server document with its own restrictive CSP. The document initializes the account-keyed APS queue and then loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. +Both rendering paths use a publisher-origin bootstrap followed by two nested `data:` documents. TSJS first loads `GET /integrations/aps/renderer?mode=data-bootstrap` with a sandbox that omits `allow-same-origin`. The bootstrap is therefore opaque and cannot read or modify the publisher document. After a nonce-bound readiness message, TSJS adds `allow-same-origin` and asks the bootstrap to navigate itself to a per-impression `data:` container. The container then creates the static inner `data:` renderer under the same permanent sandbox. -The outer iframe uses these sandbox permissions: +Both data documents are naturally opaque even with `allow-same-origin`, so the publisher cannot access them. Keeping that token on both frames also avoids WebKit propagating an opaque origin to the HTTPS creative: the creative and its same-origin descendants retain their real origin in Chromium, Firefox, and WebKit. -```text -allow-forms -allow-pointer-lock -allow-popups -allow-popups-to-escape-sandbox -allow-scripts -allow-top-navigation-by-user-activation -``` +The outer container's CSP allows child frames from `data:` and only the fully validated creative URL's exact origin. That policy is inherited by the inner data renderer and intersects with the renderer's own CSP. It permits the expected creative frame but blocks the inner renderer from navigating itself to the publisher origin before a request is made. This exact-origin boundary intentionally blocks an immediate creative-frame redirect or same-frame navigation to a different origin; validate real APS inventory for intermediate or redirect origins before rollout. User-activated top navigation and popups remain governed by the sandbox tokens. + +A network-loaded HTTPS creative does not inherit its ancestor's CSP and can create further frames. To prevent it from using a publisher-origin descendant plus an executable publisher gadget to regain access to the top page, enabling APS appends a separate `Content-Security-Policy: frame-ancestors 'self'` policy to every Trusted Server response. Browsers require every ancestor to match, so publisher documents cannot load below the opaque container or third-party creative. This secure default also prevents the APS-enabled publisher from being embedded cross-origin; publishers that require trusted external embedders need a separately reviewed ancestor-allowlist feature before enabling APS. + +Trusted Server generates independent 128-bit nonces for the container and renderer. Once the inner renderer is ready, the container transfers a dedicated `MessagePort` between trusted top-page TSJS and the inner renderer. The complete descriptor travels only over that port; the inner renderer revalidates it, rejects the publisher origin, and requires the creative origin to equal the origin bound in the container CSP. The container URL contains only that exact origin, static renderer source, and nonces—never the response envelope, bid ID, price, or creative URL path. -It deliberately omits `allow-same-origin`, so APS and bidder execution remains below an opaque-origin boundary. The renderer response repeats these restrictions with a CSP `sandbox` directive, preventing another embedding path from restoring publisher-origin execution by omitting the iframe attribute. Trusted Server generates a fresh 128-bit nonce, binds it in the iframe URL fragment before navigation, and requires the same one-time nonce in the parent message and renderer acknowledgement. Existing slot content is retained until the static renderer has accepted the descriptor and loaded the fixed runner. +The inner document initializes the account-keyed APS queue and loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. A `renderer-ready` acknowledgement still means that Amazon's runner loaded, not that the final creative painted. Existing slot content remains until that acknowledgement. The query-free `GET /integrations/aps/renderer` response remains available with its original stricter CSP and legacy message shape for cached clients and rollback. ### Direct `/auction` -The TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. +The TSJS auction client validates the typed renderer descriptor and mounts the nested data renderer in the winning slot. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. ### GAM and Universal Creative -For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program that creates the same opaque renderer iframe. +For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program with a one-shot mount capability. That program asks trusted top-page TSJS to mount the nested data renderer as a sibling of the Universal Creative iframe, outside inherited GAM and Universal Creative sandbox restrictions. -For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes the capability once, and passes the APS bid ID separately to the Amazon runner. +For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes both the renderer and mount capabilities once, and passes the APS bid ID separately to the Amazon runner. These paths do not fetch PBS Cache, fire generic APS win/billing beacons, or call `apstag.setDisplayBids()` for the Trusted Server winner. Publisher-owned native APS objects are otherwise left untouched. @@ -182,10 +179,10 @@ These paths do not fetch PBS Cache, fire generic APS win/billing beacons, or cal The publisher policy must permit the same-origin renderer route, for example: ```text -frame-src 'self' +frame-src 'self' data: ``` -Do not add `allow-same-origin` to the outer renderer sandbox. The renderer endpoint supplies its own CSP for the fixed runner and HTTPS creative resources. The same-origin renderer route inherits the publisher page scheme; use HTTPS in production. APS endpoints and third-party creative URLs always require HTTPS. +The `data:` source is required for the bootstrap's self-navigation to the opaque container. APS also adds `frame-ancestors 'self'` as an independent response policy; do not override or remove that policy. Do not weaken or bypass the initial bootstrap sandbox. TSJS adds `allow-same-origin` only when navigating away from the publisher-origin bootstrap; both final data documents remain naturally opaque. The bootstrap response supplies the resource CSP for the fixed runner and HTTPS creative resources, while the container narrows `frame-src` to the validated creative origin. The same-origin bootstrap route inherits the publisher page scheme; use HTTPS in production. APS endpoints and third-party creative URLs always require HTTPS. Before enabling script creatives, verify under the publisher's actual CSP that both iframe and script-tag creatives: @@ -235,8 +232,8 @@ Use fictional values in source-controlled configuration and fixtures. Supply con ### Winner targets but does not render -- Confirm `GET /integrations/aps/renderer` returns HTML with its CSP and `Referrer-Policy: no-referrer`. -- Confirm publisher CSP permits `frame-src 'self'`. +- Confirm `GET /integrations/aps/renderer?mode=data-bootstrap` returns HTML with its bootstrap CSP and `Referrer-Policy: no-referrer`. +- Confirm publisher CSP permits `frame-src 'self' data:`. - Confirm the GAM creative uses the supported Prebid Universal Creative bridge and the winning `hb_adid`. - For client-side `trustedServer` adapter auctions, confirm Prebid's `bidResponse` contains a generated `adId` and that the corresponding capability appears briefly in `window.tsjs.apsPrebidRenderers` before rendering. - Ensure no native APS path is trying to handle the same cohort. From 18d28a5a8fe794a7dfc6696d85f611a446e05aa6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 15 Aug 2026 09:25:49 +0530 Subject: [PATCH 246/395] Add GAM attribution bundle fallback --- .../lib/src/integrations/gpt/index.ts | 22 ++ .../lib/test/integrations/gpt/index.test.ts | 221 ++++++++++++++++++ 2 files changed, 243 insertions(+) diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index f0df35974..46acd1b0c 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -287,6 +287,8 @@ type GptWindow = Window & { __tsjs_slim_prebid_url?: string; }; +const executingPublisherScript = typeof document === 'undefined' ? null : document.currentScript; + // ------------------------------------------------------------------ // Shim implementation // ------------------------------------------------------------------ @@ -307,6 +309,25 @@ function ensureGoogleTagStub(win: GptWindow): Partial { return tag; } +function installTrustedServerPageTargeting(): void { + if (executingPublisherScript?.getAttribute('data-ts-gam-attribution') !== 'true') { + return; + } + + const win = window as GptWindow; + const tag = ensureGoogleTagStub(win); + tag.cmd!.push(() => { + try { + const gpt = win.googletag; + if (typeof gpt?.setConfig === 'function') { + gpt.setConfig({ targeting: { ts: 'true' } }); + } + } catch (error) { + log.warn('[tsjs-gpt] GAM attribution targeting failed', error); + } + }); +} + /** * Wrap a queued GPT callback to add instrumentation and future hook points. * @@ -1892,6 +1913,7 @@ if (typeof window !== 'undefined') { installGptShim(); } + installTrustedServerPageTargeting(); installTsAdInit(); installSpaAuctionHook(); installSlimPrebidLoader(); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts index e8251be89..f684d7188 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts @@ -420,6 +420,227 @@ describe('GPT shim – runtime gating', () => { }); }); +describe('GPT GAM attribution bundle fallback', () => { + interface AttributionPubAds { + getSlots: () => unknown[]; + refresh: (...args: unknown[]) => void; + } + + interface AttributionGoogleTag { + cmd: Array<() => void>; + pubads: () => AttributionPubAds; + defineSlot: (...args: unknown[]) => unknown; + display: (...args: unknown[]) => void; + getConfig?: (keys: string | string[]) => Record | undefined; + setConfig?: (config: Record) => void; + } + + type AttributionWindow = Omit & { + tsjs?: Partial; + googletag?: AttributionGoogleTag; + __tsjs_gpt_enabled?: boolean; + __tsjs_installGptShim?: unknown; + __tsjs_slim_prebid_url?: string; + }; + + let win: AttributionWindow; + let executingScript: HTMLScriptElement | null; + let originalCurrentScriptDescriptor: PropertyDescriptor | undefined; + let originalPushState: History['pushState']; + let originalReplaceState: History['replaceState']; + let addEventListenerSpy: ReturnType; + + function makeGoogleTag(overrides: Partial = {}): AttributionGoogleTag { + const pubads: AttributionPubAds = { + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + + return { + cmd: [], + pubads: vi.fn(() => pubads), + defineSlot: vi.fn(), + display: vi.fn(), + ...overrides, + }; + } + + function attributedScript(ownerDocument: Document = document): HTMLScriptElement { + const script = ownerDocument.createElement('script'); + script.id = 'trustedserver-js'; + script.setAttribute('data-ts-gam-attribution', 'true'); + return script; + } + + async function importFreshGptBundle(): Promise { + await import('../../../src/integrations/gpt/index'); + } + + beforeEach(async () => { + const guard = await importGuardModule(); + guard.resetGuardState(); + vi.resetModules(); + + win = window as AttributionWindow; + delete win.tsjs; + delete win.googletag; + delete win.__tsjs_gpt_enabled; + delete win.__tsjs_installGptShim; + delete win.__tsjs_slim_prebid_url; + + executingScript = null; + originalCurrentScriptDescriptor = Object.getOwnPropertyDescriptor(document, 'currentScript'); + Object.defineProperty(document, 'currentScript', { + configurable: true, + get: () => executingScript, + }); + + originalPushState = history.pushState; + originalReplaceState = history.replaceState; + addEventListenerSpy = vi.spyOn(window, 'addEventListener').mockImplementation(() => {}); + }); + + afterEach(async () => { + const guard = await importGuardModule(); + guard.resetGuardState(); + history.pushState = originalPushState; + history.replaceState = originalReplaceState; + if (originalCurrentScriptDescriptor) { + Object.defineProperty(document, 'currentScript', originalCurrentScriptDescriptor); + } else { + delete (document as unknown as Record).currentScript; + } + delete win.tsjs; + delete win.googletag; + delete win.__tsjs_gpt_enabled; + delete win.__tsjs_installGptShim; + delete win.__tsjs_slim_prebid_url; + vi.restoreAllMocks(); + }); + + it('reuses the existing queue and pushes exact targeting before adInit exists', async () => { + const queue: Array<() => void> = []; + const adInitAtPush: Array = []; + const arrayPush = queue.push.bind(queue); + queue.push = (...callbacks: Array<() => void>) => { + callbacks.forEach(() => adInitAtPush.push(win.tsjs?.adInit)); + return arrayPush(...callbacks); + }; + const setConfig = vi.fn(); + const tag = makeGoogleTag({ cmd: queue, setConfig }); + win.googletag = tag; + executingScript = attributedScript(); + const initialScriptCount = document.scripts.length; + const fetchSpy = vi.spyOn(globalThis, 'fetch'); + + await importFreshGptBundle(); + + expect(win.googletag!.cmd).toBe(queue); + expect(adInitAtPush[0]).toBeUndefined(); + expect(queue.length).toBeGreaterThan(0); + queue[0](); + expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }); + expect(document.scripts).toHaveLength(initialScriptCount); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it.each([ + ['no current script', null], + ['missing attribute', undefined], + ['empty attribute', ''], + ['false attribute', 'false'], + ['non-exact attribute', 'TRUE'], + ])('fails closed for %s', async (_label, attributeValue) => { + if (attributeValue === null) { + executingScript = null; + } else { + executingScript = document.createElement('script'); + if (attributeValue !== undefined) { + executingScript.setAttribute('data-ts-gam-attribution', attributeValue); + } + } + + await importFreshGptBundle(); + + expect(win.googletag).toBeUndefined(); + }); + + it('ignores a marked duplicate-ID decoy when the executing tag is unmarked', async () => { + const decoy = attributedScript(); + document.head.appendChild(decoy); + executingScript = document.createElement('script'); + executingScript.id = 'trustedserver-js'; + + await importFreshGptBundle(); + + expect(win.googletag).toBeUndefined(); + decoy.remove(); + }); + + it('characterizes a marked document.write clone as able to activate', async () => { + const clonedDocument = document.implementation.createHTMLDocument('nested clone'); + clonedDocument.write(''); + clonedDocument.close(); + executingScript = clonedDocument.querySelector('script'); + const queue: Array<() => void> = []; + const setConfig = vi.fn(); + win.googletag = makeGoogleTag({ cmd: queue, setConfig }); + + await importFreshGptBundle(); + queue[0](); + + expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }); + }); + + it.each(['missing', 'throwing'])( + 'keeps module installation working with %s setConfig', + async (setConfigMode) => { + const queue: Array<() => void> = []; + const setConfig = + setConfigMode === 'throwing' + ? vi.fn(() => { + throw new Error('publisher setConfig failed'); + }) + : undefined; + win.googletag = makeGoogleTag({ cmd: queue, setConfig }); + win.__tsjs_slim_prebid_url = 'https://cdn.example.com/slim-prebid.js'; + executingScript = attributedScript(); + + await importFreshGptBundle(); + + expect(() => [...queue].forEach((command) => command())).not.toThrow(); + expect(typeof win.tsjs?.adInit).toBe('function'); + expect(typeof win.tsjs?.scheduleInitialAdInit).toBe('function'); + expect(win.tsjs?.spaHookInstalled).toBe(true); + expect(addEventListenerSpy).toHaveBeenCalledWith('popstate', expect.any(Function)); + expect(addEventListenerSpy).toHaveBeenCalledWith('load', expect.any(Function)); + expect(addEventListenerSpy).toHaveBeenCalledWith('message', expect.any(Function)); + if (setConfig) { + expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }); + } + } + ); + + it('preserves GPT-enabled shim behavior without queuing attribution when unmarked', async () => { + const queue: Array<() => void> = []; + const setConfig = vi.fn(); + const tag = makeGoogleTag({ cmd: queue, setConfig }); + win.googletag = tag; + win.__tsjs_gpt_enabled = true; + executingScript = document.createElement('script'); + + await importFreshGptBundle(); + [...queue].forEach((command) => command()); + const guard = await importGuardModule(); + + expect(guard.isGuardInstalled()).toBe(true); + expect(win.googletag).toBe(tag); + expect(win.googletag!.cmd).toBe(queue); + expect(setConfig).not.toHaveBeenCalled(); + expect(typeof win.tsjs?.adInit).toBe('function'); + }); +}); + describe('GPT debug ADM iframe hardening', () => { it('sandbox token list omits allow-same-origin', async () => { const mod = await import('../../../src/integrations/gpt/index'); From d65f6562e4315cd7c6167526b09487db6f4d958b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 15 Aug 2026 09:31:52 +0530 Subject: [PATCH 247/395] Characterize targeting collisions --- .../src/creative_opportunities.rs | 9 ++- crates/trusted-server-core/src/publisher.rs | 11 +++- .../lib/test/integrations/gpt/ad_init.test.ts | 3 +- .../test/integrations/prebid/index.test.ts | 58 ++++++++++++++++--- 4 files changed, 71 insertions(+), 10 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index e44b0cbcf..6fc9c8dc5 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -1711,11 +1711,18 @@ mod tests { #[test] fn to_ad_slot_sets_floor_price_and_formats() { - let slot = make_slot("atf", vec!["/"]); + let mut slot = make_slot("atf", vec!["/"]); + slot.targeting + .insert("ts".to_string(), "operator-value".to_string()); let ad_slot = slot.to_ad_slot(); assert_eq!(ad_slot.id, "atf"); assert_eq!(ad_slot.floor_price, Some(0.50)); assert_eq!(ad_slot.formats.len(), 1); + assert_eq!( + ad_slot.targeting.get("ts"), + Some(&serde_json::Value::String("operator-value".to_owned())), + "should preserve operator-provided ts targeting verbatim" + ); } #[test] diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4bed98327..00c403408 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -8736,9 +8736,14 @@ mod tests { #[test] fn ad_slots_script_contains_slot_data() { - let slots = vec![make_slot()]; + let mut slot = make_slot(); + slot.targeting + .insert("ts".to_string(), "operator-value".to_string()); + let slots = vec![slot]; let config = make_config(); let script = build_ad_slots_script(&slots, &config, "/"); + let slot_json = crate::publisher::build_slot_json(&slots[0], &config, "example") + .expect("should build slot JSON"); assert!( script.contains("window.tsjs=window.tsjs||{}"), "should initialise tsjs namespace" @@ -8753,6 +8758,10 @@ mod tests { !script.contains("__ts_request_id"), "must NOT contain request_id" ); + assert_eq!( + slot_json["targeting"]["ts"], "operator-value", + "should forward operator-provided ts targeting verbatim" + ); } #[test] diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 179a810d5..7e7c6d10a 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -1874,7 +1874,7 @@ describe('installTsAdInit', () => { setTargeting: vi.fn().mockReturnThis(), clearTargeting, getSlotElementId: vi.fn().mockReturnValue('div-old-route'), - getTargeting: vi.fn().mockReturnValue([]), + getTargeting: vi.fn((key: string) => (key === 'ts' ? ['publisher-value'] : [])), }; const mockPubads = { enableSingleRequest: vi.fn(), @@ -1908,6 +1908,7 @@ describe('installTsAdInit', () => { expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); expect(clearTargeting).toHaveBeenCalledWith('pos'); + expect(clearTargeting).not.toHaveBeenCalledWith('ts'); expect(mockPubads.refresh).not.toHaveBeenCalled(); expect((window as TestWindow).tsjs!.divToSlotId).toEqual({}); expect((window as TestWindow).tsjs!.prevSlotTargetingKeys).toEqual({}); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 9f9a3f977..8ead01aa8 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -414,9 +414,11 @@ describe('prebid/installPrebidNpm', () => { delete testWindow.__tsjs_prebid_diagnostics; delete testWindow.tsjs; delete mockPbjs['__tsApsBidResponseListenerInstalled']; + delete mockPbjs.bidderSettings; }); afterEach(() => { + delete mockPbjs.bidderSettings; vi.restoreAllMocks(); }); @@ -1130,6 +1132,35 @@ describe('prebid/installPrebidNpm', () => { }); describe('requestBids shim', () => { + it('preserves publisher ts adserverTargeting while adding trustedServer settings', () => { + const publisherTargeting = [{ key: 'ts', val: () => 'publisher-value' }]; + mockPbjs.bidderSettings = { + exampleBidder: { adserverTargeting: publisherTargeting }, + }; + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ bids: [{ bidder: 'exampleBidder', params: {} }] }], + } as unknown as RequestBidsArg); + + const bidderSettings = mockPbjs.bidderSettings as { + exampleBidder: { adserverTargeting: typeof publisherTargeting }; + trustedServer: { + allowAlternateBidderCodes: boolean; + allowedAlternateBidderCodes: string[]; + }; + }; + expect(bidderSettings.exampleBidder.adserverTargeting).toBe(publisherTargeting); + expect(bidderSettings.exampleBidder.adserverTargeting[0].key).toBe('ts'); + expect(bidderSettings.exampleBidder.adserverTargeting[0].val()).toBe('publisher-value'); + expect(bidderSettings.trustedServer).toEqual( + expect.objectContaining({ + allowAlternateBidderCodes: true, + allowedAlternateBidderCodes: ['*'], + }) + ); + }); + it('injects trustedServer bidder into every ad unit', () => { const pbjs = installPrebidNpm(); @@ -1850,25 +1881,33 @@ describe('prebid/installRefreshHandler', () => { it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { const originalRefresh = vi.fn(); - const clearTargeting = vi.fn(); + const slotTargeting = new Map([ + ['ts_initial', ['1']], + ['zone', ['homepage']], + ]); + const clearTargeting = vi.fn((key: string) => { + slotTargeting.delete(key); + }); + const setTargeting = vi.fn((key: string, value: string | string[]) => { + slotTargeting.set(key, Array.isArray(value) ? value : [value]); + }); const gptSlot = { getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), - getTargeting: vi.fn((key: string) => { - if (key === 'ts_initial') return ['1']; - if (key === 'zone') return ['homepage']; - return []; - }), + getTargeting: vi.fn((key: string) => slotTargeting.get(key) ?? []), getSizes: vi.fn(() => [ { getWidth: () => 970, getHeight: () => 250 }, { getWidth: () => 728, getHeight: () => 90 }, ]), clearTargeting, + setTargeting, }; const pubads = { refresh: originalRefresh, getSlots: vi.fn(() => [gptSlot]), }; - const setTargetingForGPTAsync = vi.fn(); + const setTargetingForGPTAsync = vi.fn(() => { + gptSlot.setTargeting('ts', 'prebid-value'); + }); mockPbjs.setTargetingForGPTAsync = setTargetingForGPTAsync; testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, @@ -1918,12 +1957,17 @@ describe('prebid/installRefreshHandler', () => { expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(clearTargeting).not.toHaveBeenCalledWith('ts'); expect(originalRefresh).not.toHaveBeenCalled(); const bidsBackHandler = mockRequestBids.mock.calls[0][0].bidsBackHandler; bidsBackHandler(); expect(setTargetingForGPTAsync).toHaveBeenCalled(); + expect(setTargetingForGPTAsync.mock.invocationCallOrder[0]).toBeLessThan( + originalRefresh.mock.invocationCallOrder[0] + ); + expect(slotTargeting.get('ts')).toEqual(['prebid-value']); expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); }); From ef37c1154fc847eacfd159fba8b8cd9bba44b626 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 15 Aug 2026 09:33:56 +0530 Subject: [PATCH 248/395] Define GAM attribution streaming semantics --- crates/trusted-server-core/src/publisher.rs | 44 ++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 00c403408..d95f82567 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -7754,7 +7754,15 @@ mod tests { } fn streaming_finalize_response(params: OwnedProcessResponseParams, body: EdgeBody) -> EdgeBody { - let settings = Arc::new(create_test_settings()); + streaming_finalize_response_with_settings(params, body, create_test_settings()) + } + + fn streaming_finalize_response_with_settings( + params: OwnedProcessResponseParams, + body: EdgeBody, + settings: Settings, + ) -> EdgeBody { + let settings = Arc::new(settings); let registry = Arc::new( IntegrationRegistry::new(&settings).expect("should create integration registry"), ); @@ -7807,6 +7815,40 @@ mod tests { } } + #[test] + fn streaming_finalize_emits_gam_attribution_head_before_origin_eof() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "gpt", + &serde_json::json!({ + "enabled": true, + "gam_attribution_enabled": true + }), + ) + .expect("should insert GPT config"); + + let body = streaming_finalize_response_with_settings( + html_stream_params("", None), + origin_chunk_then_pending(bytes::Bytes::from_static( + b"

origin remains pending

", + )), + settings, + ); + let html = String::from_utf8(first_lazy_body_chunk(body).to_vec()) + .expect("should emit UTF-8 HTML"); + + assert!( + html.contains("__tsjs_gam_attribution_enabled=true"), + "first rewritten head chunk should carry the primary activation flag: {html}" + ); + assert!( + html.contains("data-ts-gam-attribution=\"true\""), + "first rewritten head chunk should authorize the bundle fallback: {html}" + ); + } + #[test] fn streaming_finalize_emits_compressed_html_before_origin_eof() { // The FCP regression from #849: the lazy body must emit its first From 17494141ca593bea370a8b34feaede9105d1588d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 15 Aug 2026 09:57:52 +0530 Subject: [PATCH 249/395] Document GAM attribution configuration --- .../tests/config_env_overlay.rs | 44 +++++++++++ .../tests/shared/script-injection.spec.ts | 1 + .../configs/trusted-server.integration.toml | 1 + docs/guide/integrations/gpt.md | 78 +++++++++++++++++-- trusted-server.example.toml | 3 + 5 files changed, 121 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-cli/tests/config_env_overlay.rs b/crates/trusted-server-cli/tests/config_env_overlay.rs index 39345137b..35263c0eb 100644 --- a/crates/trusted-server-cli/tests/config_env_overlay.rs +++ b/crates/trusted-server-cli/tests/config_env_overlay.rs @@ -29,6 +29,7 @@ ids = ["trusted_server_secrets"] "#; const REWRITE_ENV: &str = "TRUSTED_SERVER__AUCTION__REWRITE_CREATIVES"; const SANITIZE_ENV: &str = "TRUSTED_SERVER__AUCTION__SANITIZE_CREATIVES"; +const GAM_ATTRIBUTION_ENV: &str = "TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED"; struct MigratedProject { directory: TempDir, @@ -112,6 +113,49 @@ fn migrated_legacy_config_applies_rewrite_creatives_environment_override() { ); } +#[test] +fn migrated_legacy_config_applies_gam_attribution_environment_override() { + let project = migrated_legacy_project(); + let output = Command::new(env!("CARGO_BIN_EXE_ts")) + .args(["config", "push", "--adapter", "axum", "--manifest"]) + .arg(&project.manifest_path) + .arg("--app-config") + .arg(&project.config_path) + .args(["--yes", "--no-diff"]) + .current_dir(project.directory.path()) + .env(GAM_ATTRIBUTION_ENV, "true") + .output() + .expect("should run ts config push"); + + assert!( + output.status.success(), + "valid boolean overlay should push successfully: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let local_store_path = project + .directory + .path() + .join(".edgezero/local-config-trusted_server_config.json"); + let local_store: serde_json::Value = serde_json::from_str( + &fs::read_to_string(local_store_path).expect("should read pushed local config"), + ) + .expect("should parse local config store"); + let envelope_json = local_store + .as_object() + .and_then(|entries| entries.values().next()) + .and_then(serde_json::Value::as_str) + .expect("should contain a blob envelope"); + let envelope: serde_json::Value = + serde_json::from_str(envelope_json).expect("should parse blob envelope"); + + assert_eq!( + envelope["data"]["integrations"]["gpt"]["gam_attribution_enabled"], + serde_json::Value::Bool(true), + "pushed config should contain the GAM attribution environment override" + ); +} + #[test] fn migrated_legacy_config_applies_sanitize_creatives_environment_override() { let project = migrated_legacy_project(); diff --git a/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts b/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts index 3a38aa746..78059596f 100644 --- a/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts +++ b/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts @@ -9,6 +9,7 @@ test.describe("Script injection", () => { const src = await scriptTag.getAttribute("src"); expect(src).toContain("/static/tsjs="); + await expect(scriptTag).not.toHaveAttribute("data-ts-gam-attribution"); }); test("no unexpected console errors on page load", async ({ page }) => { diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml index 17d7c2713..d8e35d179 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml @@ -86,6 +86,7 @@ rewrite_sdk = true [integrations.gpt] enabled = false +gam_attribution_enabled = false script_url = "https://ads.example.com/gpt.js" cache_ttl_seconds = 3600 rewrite_script = true diff --git a/docs/guide/integrations/gpt.md b/docs/guide/integrations/gpt.md index f38f68231..174301ba1 100644 --- a/docs/guide/integrations/gpt.md +++ b/docs/guide/integrations/gpt.md @@ -53,6 +53,7 @@ Add GPT configuration to `trusted-server.toml`: ```toml [integrations.gpt] enabled = true +gam_attribution_enabled = false script_url = "https://securepubads.g.doubleclick.net/tag/js/gpt.js" cache_ttl_seconds = 3600 rewrite_script = true @@ -60,12 +61,18 @@ rewrite_script = true ### Configuration Options -| Field | Type | Required | Default | Description | -| ------------------- | ------- | -------- | ------------------------------------------------------ | ------------------------------------------ | -| `enabled` | boolean | No | `true` | Enable/disable the integration | -| `script_url` | string | No | `https://securepubads.g.doubleclick.net/tag/js/gpt.js` | URL for the GPT bootstrap script | -| `cache_ttl_seconds` | integer | No | `3600` | Cache TTL for proxied scripts (60--86400s) | -| `rewrite_script` | boolean | No | `true` | Whether to rewrite GPT script URLs in HTML | +| Field | Type | Required | Default | Description | +| ------------------------- | ------- | -------- | ------------------------------------------------------ | ----------------------------------------------------------------- | +| `enabled` | boolean | No | `true` | Enable/disable the integration | +| `gam_attribution_enabled` | boolean | No | `false` | Add fixed page-level `ts=true` targeting for GAM cohort reporting | +| `script_url` | string | No | `https://securepubads.g.doubleclick.net/tag/js/gpt.js` | URL for the GPT bootstrap script | +| `cache_ttl_seconds` | integer | No | `3600` | Cache TTL for proxied scripts (60--86400s) | +| `rewrite_script` | boolean | No | `true` | Whether to rewrite GPT script URLs in HTML | + +The environment override +`TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED` works only when +`gam_attribution_enabled` is already present under `[integrations.gpt]` in the +TOML file. EdgeZero v0.0.4 cannot create a missing configuration leaf. ## Endpoints @@ -109,6 +116,65 @@ Takes over `googletag.cmd` so every queued callback is wrapped before GPT execut - Consent gating of ad requests - Ad-unit path rewriting for A/B testing +### GAM Treatment Attribution + +Setting `gam_attribution_enabled = true` adds the fixed page-level GPT targeting +value `ts=true`. It is applied before publisher GPT initialization and remains +for the browser document's lifetime, so initial, lazy, refresh, publisher-owned, +and SPA-route requests inherit it unless another targeting consumer clears or +overrides the key. The attribution switch is independently controlled and +defaults to `false`, but the GPT integration's `enabled` master switch must also +be `true`. + +This key is distinct from the existing slot-level `ts_initial=1` value. +`ts_initial` retains its current cleanup lifecycle; Trusted Server does not +clear the page-level `ts` value during Prebid refresh or SPA cleanup. + +For an eligible publisher document whose activation script was not cloned, +`ts=true` means Trusted Server emitted the rewritten document head before the +GPT request. It does not prove that the response body completed, that a Trusted +Server bid won, or that an impression was caused by treatment. A publisher can +copy the activation script with `srcdoc` or `document.write`; treat any marker +on an unrewritten nested document as contamination, not attribution proof. + +Before enabling attribution in a cohort: + +1. Complete privacy and CSP review, create the reportable predefined `true` + value in the target GAM network, and verify the chosen GAM reporting surface + and billing approval. +2. Audit the short `ts` key across publisher GPT code, effective Prebid + `bidderSettings[*].adserverTargeting` output (including + `setTargetingForGPTAsync`), the effective creative-opportunity targeting map, + and every GAM consumer that can affect eligibility, pricing, protection, or + routing. Trusted Server accepts and forwards operator targeting verbatim; it + does not reserve, filter, or intercept a slot-level `ts` key at runtime. +3. With treatment routing stopped, deploy attribution enabled and validate + initial, lazy, refresh, publisher-owned, and SPA requests. Confirm every + excluded path reports zero marked requests, then save a short paired-report + dry run that satisfies the invariants below before starting the cohort. + +For reporting, save one exact eligible universe: GAM network, inventory units, +routes, formats, time zone, date window, metrics, and all exclusions. Report A +is the nonduplicated total for that universe. Report B uses identical filters +and metrics plus exactly `ts=true`. If Enhanced Key-Value reporting is +unavailable, unapproved, or incompatible, use an exactly filtered legacy +key-value report and never sum its repeated key-value rows. Derive control as +`A - B`, and require `0 <= B <= A` for every metric. A violation invalidates the +whole report pair; never clamp a negative result. Use the same reporting-latency +and invalid-traffic maturation window for both reports. + +GAM results are descriptive delivery attribution, not a causal treatment +effect. Aggregate monitoring and synthetic/manual samples can detect obvious +failures but cannot prove marker completeness on every production request +without request-correlated telemetry. + +For a normal rollback, first stop and verify new treatment assignment at the +router, record a clean reporting boundary, and let already-open documents drain. +Exclude the drain interval, then set `gam_attribution_enabled = false` after +marked traffic reaches zero for the agreed interval. An emergency kill may flip +the setting immediately, but the affected interval and subsequent drain must be +treated as invalid for experiment reporting. + ## Use Cases ### First-Party Ad Delivery diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 19ecda4a5..38af8a1ec 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -104,6 +104,9 @@ rewrite_sdk = true [integrations.gpt] enabled = false +# Keep this leaf present when using the corresponding EdgeZero v0.0.4 +# environment override. Attribution remains off until explicitly enabled. +gam_attribution_enabled = false script_url = "https://ads.example.com/gpt.js" cache_ttl_seconds = 3600 rewrite_script = true From ef35343fff2b6938ee2a3fdd153f19dec792eaf0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 15 Aug 2026 10:01:49 +0530 Subject: [PATCH 250/395] Use explicit GAM attribution branch --- crates/trusted-server-core/src/integrations/gpt.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 533944027..2c2c94ba7 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -491,11 +491,11 @@ impl IntegrationHeadInjector for GptIntegration { /// route changes (see `auction/endpoints.rs`). /// The `POST /auction` endpoint is not involved in scroll or refresh flows. fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - let gam_attribution_flag = self - .config - .gam_attribution_enabled - .then_some("window.__tsjs_gam_attribution_enabled=true;") - .unwrap_or_default(); + let gam_attribution_flag = if self.config.gam_attribution_enabled { + "window.__tsjs_gam_attribution_enabled=true;" + } else { + "" + }; let mut scripts = vec![ format!( From 97e097d4108a40c9bad4bf6d36bbacb3c2ce295c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 15 Aug 2026 10:12:29 +0530 Subject: [PATCH 251/395] Align GAM rollback and failure coverage --- .../integrations/gpt/gpt_bootstrap.test.ts | 16 +++++++- .../2026-07-15-gam-ts-cohort-attribution.md | 16 ++++---- ...-07-15-gam-ts-cohort-attribution-design.md | 37 ++++++++++--------- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index 4ec4110b5..96f377933 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -174,8 +174,18 @@ describe('gpt_bootstrap.js fallback', () => { const setConfig = vi.fn(() => { throw new Error('publisher setConfig failed'); }); + const disableInitialLoad = vi.fn(); + const pubads = { + disableInitialLoad, + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; const publisherCommand = vi.fn(); - (window as TestWindow).googletag = makeGoogleTag({ cmd: queue, setConfig }); + (window as TestWindow).googletag = makeGoogleTag({ + cmd: queue, + setConfig, + pubads: vi.fn(() => pubads), + }); (window as TestWindow).__tsjs_gam_attribution_enabled = true; runBootstrap(); @@ -184,6 +194,10 @@ describe('gpt_bootstrap.js fallback', () => { expect(() => [...queue].forEach((command) => command())).not.toThrow(); expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }); expect(publisherCommand).toHaveBeenCalledTimes(1); + expect(typeof (window as TestWindow).tsjs!.adInit).toBe('function'); + pubads.disableInitialLoad(); + expect(disableInitialLoad).toHaveBeenCalledTimes(1); + expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).toBe(true); }); it('still tracks the wrapped legacy disableInitialLoad path', () => { diff --git a/docs/superpowers/plans/2026-07-15-gam-ts-cohort-attribution.md b/docs/superpowers/plans/2026-07-15-gam-ts-cohort-attribution.md index d7763e046..eaf07ca51 100644 --- a/docs/superpowers/plans/2026-07-15-gam-ts-cohort-attribution.md +++ b/docs/superpowers/plans/2026-07-15-gam-ts-cohort-attribution.md @@ -905,8 +905,9 @@ This task must not add buffering or adapter logic. - saved report pairs are exported after the same reporting-latency and invalid-traffic window, and monitoring/synthetic samples do not prove per-response marker completeness; - - normal rollback stops and verifies routing before flipping the setting, - then excludes the open-document drain interval; + - normal rollback stops and verifies routing, records the boundary, keeps + attribution enabled through the excluded open-document drain, and flips the + setting only after marked traffic reaches zero; - an emergency kill flips immediately and invalidates the affected/drain interval. @@ -1018,7 +1019,7 @@ This task must not add buffering or adapter logic. git log --oneline --decorate -8 ``` - Expected: a clean worktree, seven focused implementation commits, and no + Expected: a clean worktree, focused implementation commits, and no production changes to creative-opportunity filtering, Prebid interception, streaming buffering, or adapter behavior. @@ -1052,10 +1053,11 @@ remain separate from Tasks 1-8 because they require publisher/router/GAM access. `0 <= Report B <= Report A` for every selected metric. - [ ] Start the sticky treatment cohort only after every gate passes; use GAM share versus router allocation only as a diagnostic. -- [ ] For normal rollback, close the clean report boundary, stop and verify new - treatment routing, disable attribution, and exclude the marked-document - drain interval. For an emergency kill, disable immediately and invalidate - the affected plus drain intervals. +- [ ] For normal rollback, stop and verify new treatment routing, close the + clean report boundary, keep attribution enabled while marked documents + drain through the excluded interval, and disable only after marked traffic + reaches zero for the agreed interval. For an emergency kill, disable + immediately and invalidate the affected plus drain intervals. ## Definition of done diff --git a/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md b/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md index 5e9172d4c..38ffa7ef4 100644 --- a/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md +++ b/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md @@ -986,19 +986,20 @@ deployment: diagnostic before interpreting descriptive delivery results. Rollback is ordered so newly routed treatment traffic cannot become unmarked -control. First record the last clean reporting boundary and stop new treatment -assignment/routing. Verify through router or access logs that routing stopped; -then set `gam_attribution_enabled = false` and deploy the kill switch. Fresh -Trusted Server documents must keep normal GPT behavior while omitting the -marker. Already-open documents—including long-lived SPA sessions and any marked -document restored from a cache—retain page-level targeting and may continue -issuing marked lazy or refreshed requests. Exclude the entire post-boundary -drain interval from both cohorts. The drain ends only after router/access logs -and GAM show no remaining `ts=true` traffic for one complete, runbook-defined -reporting interval and a fresh synthetic navigation confirms that new documents -are unmarked. If marked traffic persists, the interval remains excluded rather -than being inferred as control. Historical GAM rows remain valid, and the GAM -key may stay defined and reportable for historical analysis. +control. First stop new treatment assignment/routing and verify through router +or access logs that routing stopped, then record the last clean reporting +boundary. Keep `gam_attribution_enabled = true` while already-open +documents—including long-lived SPA sessions and any marked document restored +from a cache—drain; they retain page-level targeting and may continue issuing +marked lazy or refreshed requests. Exclude the entire post-boundary drain +interval from both cohorts. The drain ends only after router/access logs and GAM +show no remaining `ts=true` traffic for one complete, runbook-defined reporting +interval. Then set `gam_attribution_enabled = false`, deploy the kill switch, +and use a fresh synthetic Trusted Server navigation to confirm normal GPT +behavior while the marker is absent. If marked traffic persists, keep +attribution enabled and the interval excluded rather than inferring it as +control. Historical GAM rows remain valid, and the GAM key may stay defined and +reportable for historical analysis. If an active privacy, targeting-collision, or ad-delivery incident requires an immediate kill, deploy `gam_attribution_enabled = false` without waiting for @@ -1133,8 +1134,8 @@ baseline limitation and requires coverage checks. 13. CSP-compatible inline execution is proven before launch. A fallback-only page remains attributed to treatment but raises an incident and cannot be treated as a healthy experiment page. -14. Normal rollback stops and verifies new treatment routing before deploying - the attribution kill switch, and reporting excludes already-open or cached - marked documents until observed traffic drains according to the documented - boundary rule. An emergency kill invalidates the affected and drain windows - instead of treating newly unmarked traffic as control. +14. Normal rollback stops and verifies new treatment routing, records the clean + boundary, and keeps attribution enabled until already-open or cached marked + documents drain according to the documented rule. Only then is the kill + switch deployed. An emergency kill invalidates the affected and drain + windows instead of treating newly unmarked traffic as control. From 54203900d8fb7da9338a39c5e47822d55aba6219 Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:59:57 -0700 Subject: [PATCH 252/395] Sync EdgeZero to latest deploy-actions tip and adopt config gc Re-resolve the six edgezero-* deps from bb441162 to 908e229a (current tip of feature/edgezero-deploy-actions, PR #316), and adapt the ts CLI to its surface changes: - Wire the new `ts config gc` subcommand (reclaims orphaned config-store chunk entries) to edgezero_cli::run_config_gc, with parse coverage for the preview default, destructive --yes/--older-than sweep, and the --dry-run/--yes conflict. - Lock the hardened deploy staging behavior: --stage was renamed to --staging and deploy passthrough is now last=true, so a stray --stage fails closed at parse time instead of routing a staging-intended deploy to production. Add tests for the rejection and for post---- passthrough capture. --- Cargo.lock | 33 +++++++---- crates/trusted-server-cli/src/run.rs | 88 +++++++++++++++++++++++++++- 2 files changed, 108 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5caa1a7a..32e0bd696 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,7 +767,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -1398,7 +1398,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "toml", ] @@ -1406,7 +1406,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "anyhow", "async-trait", @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "anyhow", "async-trait", @@ -1457,7 +1457,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "anyhow", "async-stream", @@ -1486,7 +1486,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "anyhow", "async-trait", @@ -1513,7 +1513,7 @@ dependencies = [ [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "chrono", "clap", @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "anyhow", "async-compression", @@ -1569,14 +1569,14 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "log", "proc-macro2", "quote", "serde", "serde_json", - "syn 2.0.118", + "syn 3.0.3", "toml", "validator", ] @@ -4769,6 +4769,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -5908,7 +5919,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 7374c56a7..a395281be 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -2,8 +2,8 @@ use std::process; use clap::{Parser, Subcommand}; use edgezero_cli::args::{ - ActiveVersionArgs, AuthArgs, BuildArgs, ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, - DeployArgs, HealthcheckArgs, ProvisionArgs, RollbackArgs, ServeArgs, + ActiveVersionArgs, AuthArgs, BuildArgs, ConfigDiffArgs, ConfigGcArgs, ConfigPushArgs, + ConfigValidateArgs, DeployArgs, HealthcheckArgs, ProvisionArgs, RollbackArgs, ServeArgs, }; use trusted_server_core::config::TrustedServerAppConfig; @@ -55,6 +55,8 @@ enum ConfigCommand { Init(ConfigInitArgs), /// Diff `trusted-server.toml` against the live `EdgeZero` config. Diff(ConfigDiffArgs), + /// Reclaim orphaned chunk entries leaked from prior oversized pushes. + Gc(ConfigGcArgs), /// Push `trusted-server.toml` as a blob envelope through `EdgeZero`. Push(ConfigPushArgs), /// Validate `edgezero.toml` and the typed Trusted Server config. @@ -102,6 +104,7 @@ fn dispatch(args: Args) -> Result<(), String> { Err(err) => Err(err), } } + Command::Config(ConfigCommand::Gc(args)) => edgezero_cli::run_config_gc(&args), Command::Config(ConfigCommand::Push(args)) => { edgezero_cli::run_config_push_typed::(&args) } @@ -299,6 +302,39 @@ mod tests { assert!(deploy.staging); } + #[test] + fn deploy_rejects_renamed_stage_flag_before_separator() { + // `--stage` was renamed to `--staging`, and adapter passthrough is + // `last = true` (only captured after `--`). A stray `--stage` before the + // separator must fail closed at parse time rather than being swallowed as + // passthrough, which would leave `staging` false and route a + // staging-intended deploy to production. + Args::try_parse_from(["ts", "deploy", "--adapter", "fastly", "--stage"]) + .expect_err("should reject the renamed-away --stage flag, not route it to production"); + } + + #[test] + fn deploy_captures_adapter_passthrough_after_separator() { + let args = parse(&[ + "ts", + "deploy", + "--adapter", + "fastly", + "--", + "--comment", + "ci", + ]); + let Command::Deploy(deploy) = args.command else { + panic!("expected deploy command"); + }; + assert!(!deploy.staging, "should default to a production deploy"); + assert_eq!( + deploy.adapter_args, + vec!["--comment", "ci"], + "should capture args after -- as adapter passthrough" + ); + } + #[test] fn parses_audit_with_default_outputs() { let args = parse(&["ts", "audit", "https://publisher.example"]); @@ -438,6 +474,54 @@ mod tests { assert!(!diff.no_env); } + #[test] + fn config_gc_previews_by_default() { + let args = parse(&["ts", "config", "gc", "--adapter", "fastly"]); + let Command::Config(ConfigCommand::Gc(gc)) = args.command else { + panic!("expected config gc command"); + }; + assert_eq!(gc.adapter, "fastly"); + assert_eq!( + gc.older_than, None, + "should not require an older-than window to preview" + ); + assert!(!gc.dry_run); + assert!(!gc.no_env); + } + + #[test] + fn config_gc_parses_destructive_sweep() { + let args = parse(&[ + "ts", + "config", + "gc", + "--adapter", + "fastly", + "--yes", + "--older-than", + "7d", + ]); + let Command::Config(ConfigCommand::Gc(gc)) = args.command else { + panic!("expected config gc command"); + }; + assert!(gc.yes); + assert_eq!(gc.older_than, Some("7d".to_owned())); + } + + #[test] + fn config_gc_rejects_dry_run_with_yes() { + Args::try_parse_from([ + "ts", + "config", + "gc", + "--adapter", + "fastly", + "--dry-run", + "--yes", + ]) + .expect_err("should reject conflicting --dry-run and --yes"); + } + #[test] fn config_validate_uses_edgezero_app_config_flag() { let args = parse(&[ From fe5767e61aa8cdbeb17cb1b2b5e59a00a03640d1 Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Sun, 16 Aug 2026 00:14:38 -0700 Subject: [PATCH 253/395] Sync EdgeZero to deploy-actions tip 5f3d648c Re-resolve the six edgezero-* deps from 908e229a to 5f3d648c (current tip of feature/edgezero-deploy-actions, PR #316). The upstream change is an internal review-addressing pass (redact config-store errors, fix version parse, log cleanup, docs) confined to the Fastly adapter CLI; no ts CLI surface change, so no run.rs adaptation is needed. --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ea08f4cc..5e3388bd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,7 +767,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -1398,7 +1398,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "toml", ] @@ -1406,7 +1406,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1457,7 +1457,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-stream", @@ -1486,7 +1486,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1513,7 +1513,7 @@ dependencies = [ [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "chrono", "clap", @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-compression", @@ -1569,7 +1569,7 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "log", "proc-macro2", @@ -5920,7 +5920,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] From 31de3f63298c15b6875f5e6753a7ead13879e24c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 10:22:55 +0530 Subject: [PATCH 254/395] Fix ad-template CLI section rendering and harden the audit commands The CLI still called `resolved_gam_unit_path`, which core replaced with the path-aware `render_gam_unit_path` when `{section}` templating landed, so the crate no longer compiled. Both call sites now derive the section through `CreativeOpportunitiesConfig::section_for_path` and render the template, and `ExpectedSlot`/`ConfiguredJson` carry an optional unit path so an over-limit dynamic render is reported rather than silently matched against the wrong unit. Also resolves the outstanding review findings on these paths: - Write the operator config through a same-directory temp file, fsync, and rename, so a failed write cannot truncate `trusted-server.toml`. - Validate TLS certificates in both audit browser sessions; opting out now requires `--danger-accept-invalid-certs`. - Refuse a redirect that leaves the requested origin during verify unless `--allow-cross-origin-redirect` is passed, so another origin's evidence cannot satisfy `--strict`. - Reject page patterns the runtime cannot compile before they reach the file, through a new shared `compile_page_pattern` in core. - Reject `creative_opportunities` declared in a form the line-based splice cannot edit, instead of appending a duplicate table. - Drop non-integer GPT sizes in the collector so one fluid size cannot fail deserialization of the whole evidence payload. - Escape control characters in page-controlled text written to the terminal. --- .../src/ad_templates/compare.rs | 54 ++++- .../src/ad_templates/expected.rs | 86 +++++++- .../src/ad_templates/output.rs | 77 ++++++- .../commands/audit/ad_template_collector.js | 34 ++- .../src/commands/audit/ad_templates.rs | 195 +++++++++++++++-- .../src/commands/audit/browser.rs | 31 ++- .../src/commands/audit/collector.rs | 9 + .../audit/generate/browser_collector.rs | 5 + .../src/commands/audit/generate/mod.rs | 203 +++++++++++++++++- .../src/commands/audit/generate/slot_toml.rs | 141 +++++++++++- .../src/commands/audit/mod.rs | 7 + .../src/commands/audit/page.rs | 13 +- .../src/commands/config/ad_templates.rs | 35 ++- .../src/creative_opportunities.rs | 71 +++--- docs/guide/cli.md | 21 ++ 15 files changed, 903 insertions(+), 79 deletions(-) diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs index 18e47fb0b..215a6a300 100644 --- a/crates/trusted-server-cli/src/ad_templates/compare.rs +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -251,13 +251,27 @@ pub fn compare_page_evidence( for slot in expected { let resolved = resolve_dom(&evidence.dom_ids, &slot.div_id); let resolved_id = resolved.map(|dom| dom.dom_id.clone()); - let gpt_idx = evidence.gpt_slots.iter().position(|gpt| { - gpt.gam_unit_path == slot.gam_unit_path - && gpt_div_matches(&gpt.div_id, slot, resolved_id.as_deref()) + // An unrenderable (`None`) configured path can never match live GPT + // evidence; matching on anything else would confirm the wrong unit. + let gpt_idx = slot.gam_unit_path.as_deref().and_then(|unit_path| { + evidence.gpt_slots.iter().position(|gpt| { + gpt.gam_unit_path == unit_path + && gpt_div_matches(&gpt.div_id, slot, resolved_id.as_deref()) + }) }); let banner = banner_sizes(slot); let mut warnings = Vec::new(); + if slot.gam_unit_path.is_none() { + warnings.push(warning( + "gam_unit_path_unrenderable", + format!( + "slot `{}` gam_unit_path template renders past GAM's unit-path byte limit \ + for this page's section; the runtime rejects this config", + slot.id + ), + )); + } let (status, dom_for_evidence, gpt_for_evidence, phase) = if let Some(idx) = gpt_idx { consumed_gpt[idx] = true; @@ -434,7 +448,7 @@ mod tests { ExpectedSlot { id: id.to_string(), div_id: div_id.to_string(), - gam_unit_path: gam_unit_path.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), formats: sizes .iter() .map(|&(width, height)| ExpectedFormat { @@ -453,7 +467,7 @@ mod tests { ExpectedSlot { id: id.to_string(), div_id: div_id.to_string(), - gam_unit_path: gam_unit_path.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), formats: vec![ExpectedFormat { width: 0, height: 0, @@ -487,6 +501,36 @@ mod tests { ); } + #[test] + fn unrenderable_gam_unit_path_never_confirms() { + let mut expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + expected.gam_unit_path = None; + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Partial, + "an unrenderable configured path must not confirm against GPT evidence" + ); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "gam_unit_path_unrenderable"), + "should explain why the slot cannot be confirmed" + ); + } + #[test] fn dom_only_is_partial() { let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs index dd8e1055d..549ec0a57 100644 --- a/crates/trusted-server-cli/src/ad_templates/expected.rs +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -24,8 +24,14 @@ pub struct ExpectedSlot { pub id: String, /// Resolved HTML `div` element ID (override or the slot id). pub div_id: String, - /// Resolved GAM unit path (override or `//`). - pub gam_unit_path: String, + /// Resolved GAM unit path: the rendered `gam_unit_path` template (or + /// `//` when the slot has none). + /// + /// `None` when a dynamic template renders beyond GAM's unit-path byte limit + /// for this path's section. Runtime validation rejects such a config, so + /// this only occurs for a config that would fail to load; the slot is then + /// reported unconfirmable rather than matched against a wrong path. + pub gam_unit_path: Option, /// Configured ad formats. pub formats: Vec, /// Configured provider names, in `aps`, `prebid` order. @@ -53,16 +59,22 @@ pub struct ExpectedFormat { /// Uses [`match_slots`] so glob semantics stay identical to the runtime, and /// preserves configured slot order. `path` is assumed already normalized via /// [`normalize_path_or_url`]. +/// +/// `gam_unit_path` templates are rendered against the section the runtime would +/// derive from `path` (per the config's `section_root`/`section_segment` +/// policy), so `{section}`-bearing configs project the same unit path the live +/// page requests. // Shared projection used by the audit verifier; the static commands match slots // directly against the runtime matcher. #[must_use] pub fn expected_slots_for_path(path: &str, config: &CreativeOpportunitiesConfig) -> ExpectedSlots { + let section = config.section_for_path(path); let slots = match_slots(&config.slot, path) .into_iter() .map(|slot| ExpectedSlot { id: slot.id.clone(), div_id: slot.resolved_div_id().to_string(), - gam_unit_path: slot.resolved_gam_unit_path(&config.gam_network_id), + gam_unit_path: slot.render_gam_unit_path(&config.gam_network_id, §ion), formats: slot .formats .iter() @@ -182,7 +194,10 @@ mod tests { ["atf"] ); assert_eq!(expected.slots[0].div_id, "ad-atf-"); - assert_eq!(expected.slots[0].gam_unit_path, "/123/news/atf"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/123/news/atf") + ); assert_eq!(expected.slots[0].providers, ["prebid"]); assert_eq!( expected.slots[0].formats, @@ -208,10 +223,71 @@ mod tests { let expected = expected_slots_for_path("/", &config); assert_eq!(expected.slots[0].div_id, "footer"); - assert_eq!(expected.slots[0].gam_unit_path, "/42/footer"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/42/footer") + ); assert!(expected.slots[0].providers.is_empty()); } + #[test] + fn expected_slots_render_section_templates_per_path() { + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\", \"/news\", \"/news/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + // A path with a section segment renders that segment. + assert_eq!( + expected_slots_for_path("/news/story", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/news"), + "a section template should render the path's section" + ); + // The site root falls back to the configured section_root. + assert_eq!( + expected_slots_for_path("/", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/homepage"), + "the root path should render section_root" + ); + } + + #[test] + fn expected_slots_report_unrenderable_dynamic_template_as_none() { + // A `{section}` template that renders past GAM's 100-byte unit-path + // limit. `validate_runtime` rejects this config, so the verifier reports + // the slot as unconfirmable rather than matching a truncated path. + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{section}/{section}\"\n\ + page_patterns = [\"/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let long_path = format!("/{}", "a".repeat(60)); + let expected = expected_slots_for_path(&long_path, &config); + + assert_eq!( + expected.slots[0].gam_unit_path, None, + "an over-limit dynamic render should project as None" + ); + } + #[test] fn normalize_path_or_url_strips_query_and_fragment() { assert_eq!( diff --git a/crates/trusted-server-cli/src/ad_templates/output.rs b/crates/trusted-server-cli/src/ad_templates/output.rs index 51658d04c..9a1190652 100644 --- a/crates/trusted-server-cli/src/ad_templates/output.rs +++ b/crates/trusted-server-cli/src/ad_templates/output.rs @@ -14,10 +14,44 @@ reason = "wire model assembled by the audit verifier in a later task" )] +use std::borrow::Cow; + use serde::{Deserialize, Serialize}; use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; +/// Escapes control characters in page-controlled text bound for a terminal. +/// +/// Page titles and collector warning messages are attacker-controlled: an +/// audited page can put ANSI/OSC escape sequences in `document.title` and drive +/// the operator's terminal (cursor movement, clipboard writes, forged output) +/// when the value is printed verbatim. Every C0 control (including ESC), DEL, +/// and the C1 range are rendered as `\u{XXXX}` so the text stays inert. JSON +/// output is unaffected — `serde_json` escapes these already. +/// +/// Returns a borrowed `Cow` when the input needs no escaping. +#[must_use] +pub fn escape_terminal_text(value: &str) -> Cow<'_, str> { + if !value.chars().any(is_terminal_control) { + return Cow::Borrowed(value); + } + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + if is_terminal_control(ch) { + escaped.push_str(&format!("\\u{{{:04X}}}", ch as u32)); + } else { + escaped.push(ch); + } + } + Cow::Owned(escaped) +} + +/// Whether `ch` can act as a terminal control code (C0, DEL, or C1). +fn is_terminal_control(ch: char) -> bool { + let code = ch as u32; + code < 0x20 || (0x7f..=0x9f).contains(&code) +} + /// Confirmation status for a single configured slot. #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] @@ -175,8 +209,9 @@ pub struct SlotJson { pub struct ConfiguredJson { /// Resolved div element ID. pub div_id: String, - /// Resolved GAM unit path. - pub gam_unit_path: String, + /// Resolved GAM unit path, or `null` when a dynamic template renders past + /// GAM's unit-path byte limit for this page's section. + pub gam_unit_path: Option, /// Configured formats. pub formats: Vec, /// Configured provider names. @@ -260,7 +295,7 @@ impl VerificationReport { phase: EvidencePhaseJson::InitialLoad, configured: ConfiguredJson { div_id: "ad-atf-".to_string(), - gam_unit_path: "/123/news/atf".to_string(), + gam_unit_path: Some("/123/news/atf".to_string()), formats: vec![FormatJson { width: 300, height: 250, @@ -324,6 +359,42 @@ impl VerificationReport { mod tests { use super::*; + #[test] + fn escape_terminal_text_passes_through_ordinary_titles() { + assert!( + matches!( + escape_terminal_text("Example News — Story"), + Cow::Borrowed(_) + ), + "text with no control characters should not allocate" + ); + assert_eq!( + escape_terminal_text("Example News — Story"), + "Example News — Story" + ); + } + + #[test] + fn escape_terminal_text_neutralizes_control_sequences() { + // ESC-based CSI/OSC sequences and a raw newline are the terminal-driving + // primitives a hostile page would put in `document.title`. + assert_eq!( + escape_terminal_text("a\u{1b}]0;pwned\u{7}b"), + "a\\u{001B}]0;pwned\\u{0007}b", + "ESC and BEL should be rendered inert" + ); + assert_eq!( + escape_terminal_text("line\nforged: ok"), + "line\\u{000A}forged: ok", + "a newline should not let a title forge an output line" + ); + assert_eq!( + escape_terminal_text("del\u{7f}c1\u{9b}"), + "del\\u{007F}c1\\u{009B}", + "DEL and the C1 range should be escaped too" + ); + } + #[test] fn verification_json_contains_gate_state_and_extra_evidence() { let result = VerificationReport::example_confirmed_with_extra_evidence(); diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js index c01074376..1133d46b6 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -25,6 +25,16 @@ function __ts_push(list, entry) { if (list.length < __ts_max_entries) list.push(entry) } +// GPT sizes reach Rust as u32 pairs, so anything non-integral (fluid slots, +// NaN, negative or fractional dimensions) must be dropped here — a single bad +// pair would fail deserialization of the whole evidence payload and discard +// every other slot's otherwise valid evidence. +function __ts_size_pair(width, height) { + if (!Number.isInteger(width) || !Number.isInteger(height)) return null + if (width < 0 || height < 0) return null + return [width, height] +} + function __ts_normalize_sizes(sizes) { const out = [] if (!Array.isArray(sizes)) return out @@ -32,8 +42,9 @@ function __ts_normalize_sizes(sizes) { const pairs = typeof sizes[0] === "number" ? [sizes] : sizes for (const size of pairs) { if (out.length >= __ts_max_entries) break - if (Array.isArray(size) && typeof size[0] === "number" && typeof size[1] === "number") { - out.push([size[0], size[1]]) + const pair = Array.isArray(size) ? __ts_size_pair(size[0], size[1]) : null + if (pair) { + out.push(pair) } else { __ts_push(__ts_ev.warnings, { code: "fluid_size_ignored", @@ -148,10 +159,21 @@ window.__tsCollectAdTemplateEvidence = function () { const sizes = [] for (const size of rawSizes) { if (sizes.length >= __ts_max_entries) break - if (size && typeof size.getWidth === "function") { - sizes.push([size.getWidth(), size.getHeight()]) - } else if (Array.isArray(size) && typeof size[0] === "number") { - sizes.push([size[0], size[1]]) + let pair = null + if (size && typeof size.getWidth === "function" && typeof size.getHeight === "function") { + // A fluid GPT size answers getWidth()/getHeight() with a + // non-numeric value rather than throwing. + pair = __ts_size_pair(size.getWidth(), size.getHeight()) + } else if (Array.isArray(size)) { + pair = __ts_size_pair(size[0], size[1]) + } + if (pair) { + sizes.push(pair) + } else { + __ts_push(__ts_ev.warnings, { + code: "fluid_size_ignored", + message: "non-numeric GPT size ignored", + }) } } const exists = __ts_ev.gpt_slots.some( diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs index 8fd72b58c..8250e1cde 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -20,7 +20,7 @@ use crate::ad_templates::expected::{ExpectedSlot, expected_slots_for_path, norma use crate::ad_templates::output::{ ConfiguredJson, EvidencePhaseJson, ExtraEvidenceJson, FormatJson, GateState, Gates, GptEvidenceJson, PageJson, RuntimeAdStackExpectedJson, SlotEvidenceJson, SlotJson, SlotStatus, - VerificationReport, Warning, + VerificationReport, Warning, escape_terminal_text, }; use crate::commands::audit::AuditAdTemplatesVerifyArgs; use crate::commands::audit::collector::{ @@ -41,8 +41,11 @@ pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result<(), String loaded.settings.creative_opportunities.as_ref(), loaded.settings.auction.enabled, &args.urls, - args.strict, - args.scroll, + VerifyOptions { + strict: args.strict, + scroll: args.scroll, + allow_cross_origin_redirect: args.allow_cross_origin_redirect, + }, &args.cookies, ); @@ -61,6 +64,17 @@ pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result<(), String } } +/// Run-level verification switches. +#[derive(Debug, Clone, Copy)] +struct VerifyOptions { + /// Exit non-zero when a matched slot is missing or only partially confirmed. + strict: bool, + /// Perform a deterministic scroll pass after the initial settle. + scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + allow_cross_origin_redirect: bool, +} + /// Builds the verification report for `urls` using `collector`. /// /// `creative` is the effective `[creative_opportunities]` config (if any) and @@ -70,8 +84,7 @@ fn build_report( creative: Option<&CreativeOpportunitiesConfig>, auction_enabled: bool, urls: &[url::Url], - strict: bool, - scroll: bool, + options: VerifyOptions, cookies: &[(String, String)], ) -> VerificationReport { let init_script = build_init_script(creative); @@ -84,7 +97,7 @@ fn build_report( let request = BrowserCollectRequest { url: url.clone(), init_scripts: init_script.clone().into_iter().collect(), - scroll, + scroll: options.scroll, collect_ad_evidence: true, cookies: cookies.to_vec(), }; @@ -94,9 +107,20 @@ fn build_report( any_error = true; pages.push(error_page(url, &message)); } + // Slots are matched on the *final* path, so a redirect to a + // different origin would let an unrelated site's evidence satisfy + // `--strict` — and the path-equality redirect warning would not even + // fire when the paths happen to agree. Reject unless opted in. + Ok(collected) + if !options.allow_cross_origin_redirect + && origin_changed(url, &collected.final_url) => + { + any_error = true; + pages.push(cross_origin_page(url, &collected.final_url)); + } Ok(collected) => { let (page, strict_failed) = build_page(url, &collected, creative, auction_enabled); - if strict && strict_failed { + if options.strict && strict_failed { any_strict_fail = true; } pages.push(page); @@ -104,15 +128,20 @@ fn build_report( } } - let ok = !(any_error || (strict && any_strict_fail)); + let ok = !(any_error || (options.strict && any_strict_fail)); VerificationReport { ok, - strict, + strict: options.strict, pages, warnings: Vec::new(), } } +/// Whether navigation left the requested URL's origin (scheme, host, or port). +fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { + requested.origin() != final_url.origin() +} + /// Builds the read-only collector init script from the configured slots. fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Option { let config = AdTemplateCollectorConfig { @@ -227,6 +256,37 @@ fn error_page(requested: &url::Url, message: &str) -> PageJson { } } +/// Builds a page-level cross-origin-redirect refusal. +/// +/// The final URL is reported so the operator can re-run against it explicitly +/// (or pass `--allow-cross-origin-redirect`) once they have confirmed it is +/// their own property. +fn cross_origin_page(requested: &url::Url, final_url: &url::Url) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: None, + error: Some(Warning { + code: "cross_origin_redirect".to_string(), + message: format!( + "navigation left the requested origin ({} -> {}); \ + evidence from another origin is not accepted as verification. \ + Re-run against the final URL, or pass --allow-cross-origin-redirect", + requested.origin().ascii_serialization(), + final_url.origin().ascii_serialization(), + ), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + fn empty_evidence() -> BrowserAdEvidence { BrowserAdEvidence { dom_ids: Vec::new(), @@ -325,10 +385,29 @@ fn write_json(out: &mut dyn Write, report: &VerificationReport) -> Result<(), St } fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + // Warning codes and messages can originate in the audited page (the + // collector forwards `String(error)` from page scripts), so escape control + // characters before writing them to the operator's terminal. + let write_warning = |out: &mut dyn Write, indent: &str, warning: &Warning| { + writeln!( + out, + "{indent}warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(write_err) + }; + for page in &report.pages { writeln!(out, "url: {}", page.url).map_err(write_err)?; if let Some(error) = &page.error { - writeln!(out, " error [{}]: {}", error.code, error.message).map_err(write_err)?; + writeln!( + out, + " error [{}]: {}", + escape_terminal_text(&error.code), + escape_terminal_text(&error.message) + ) + .map_err(write_err)?; continue; } if let Some(path) = &page.path { @@ -338,13 +417,11 @@ fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), S writeln!(out, " slot {}: {}", slot.id, status_label(slot.status)) .map_err(write_err)?; for warning in &slot.warnings { - writeln!(out, " warning [{}]: {}", warning.code, warning.message) - .map_err(write_err)?; + write_warning(out, " ", warning)?; } } for warning in &page.warnings { - writeln!(out, " warning [{}]: {}", warning.code, warning.message) - .map_err(write_err)?; + write_warning(out, " ", warning)?; } } writeln!(out, "ok: {}", report.ok).map_err(write_err) @@ -449,6 +526,24 @@ mod tests { auction_enabled: bool, strict: bool, urls: &[&str], + ) -> VerificationReport { + report_for_with_options( + collector, + auction_enabled, + urls, + VerifyOptions { + strict, + scroll: false, + allow_cross_origin_redirect: false, + }, + ) + } + + fn report_for_with_options( + collector: &dyn AuditCollector, + auction_enabled: bool, + urls: &[&str], + options: VerifyOptions, ) -> VerificationReport { let config = news_config(); let parsed: Vec = urls @@ -460,8 +555,7 @@ mod tests { Some(&config), auction_enabled, &parsed, - strict, - false, + options, &[], ) } @@ -487,6 +581,75 @@ mod tests { ); } + #[test] + fn cross_origin_redirect_is_rejected_even_when_paths_match() { + // Same path on a different origin: the redirect warning would not fire, + // so without the origin check this unrelated page's evidence would + // satisfy --strict. + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://impostor.example.net/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!(!report.ok, "a cross-origin redirect must not report ok"); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][0]["error"]["code"], "cross_origin_redirect"); + assert!( + json["pages"][0]["slots"] + .as_array() + .expect("slots array") + .is_empty(), + "off-origin evidence must not be reported as slot verification" + ); + } + + #[test] + fn cross_origin_redirect_is_accepted_with_explicit_opt_in() { + let collector = FakeCollector::page( + "https://example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for_with_options( + &collector, + true, + &["https://example.com/news/story"], + VerifyOptions { + strict: true, + scroll: false, + allow_cross_origin_redirect: true, + }, + ); + + assert!( + report.ok, + "an opted-in apex -> www redirect should verify normally" + ); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn same_origin_path_redirect_still_verifies() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, true, &["https://www.example.com/"]); + + assert!( + report.ok, + "a same-origin redirect should still be verified, not refused" + ); + } + #[test] fn confirmed_page_is_ok_in_default_mode() { let collector = FakeCollector::page( diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs index a67591b59..ab998adec 100644 --- a/crates/trusted-server-cli/src/commands/audit/browser.rs +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -61,6 +61,8 @@ pub struct BrowserCollector { settle_quiet: Duration, /// Hard cap on settling. settle_max: Duration, + /// Navigate to origins with invalid TLS certificates (dangerous opt-in). + accept_invalid_certs: bool, } impl Default for BrowserCollector { @@ -77,6 +79,7 @@ impl BrowserCollector { chrome: None, settle_quiet: Duration::from_millis(DEFAULT_SETTLE_QUIET_MS), settle_max: Duration::from_millis(DEFAULT_SETTLE_MAX_MS), + accept_invalid_certs: false, } } @@ -87,6 +90,7 @@ impl BrowserCollector { chrome: opts.chrome.clone(), settle_quiet: Duration::from_millis(opts.settle_quiet_ms), settle_max: Duration::from_millis(opts.settle_max_ms), + accept_invalid_certs: opts.danger_accept_invalid_certs, } } } @@ -206,8 +210,17 @@ impl AuditCollector for BrowserCollector { // so audit output stays clean, then restore the prior threshold. let previous_level = log::max_level(); log::set_max_level(log::LevelFilter::Error); - let result = runtime - .block_on(async move { collect(&chrome, profile.path(), request, settle).await }); + let accept_invalid_certs = self.accept_invalid_certs; + let result = runtime.block_on(async move { + collect( + &chrome, + profile.path(), + request, + settle, + accept_invalid_certs, + ) + .await + }); log::set_max_level(previous_level); result } @@ -219,10 +232,20 @@ async fn collect( profile_dir: &std::path::Path, request: BrowserCollectRequest, settle_config: SettleConfig, + accept_invalid_certs: bool, ) -> Result { - let config = BrowserConfig::builder() + // chromiumoxide defaults to ignoring TLS errors. The audit sends + // operator-supplied session cookies and treats what it reads back as + // verification evidence, so a certificate-invalid impersonator could both + // harvest the session and fabricate the evidence. Validate certificates + // unless the operator explicitly opts out. + let mut builder = BrowserConfig::builder() .chrome_executable(chrome) - .user_data_dir(profile_dir) + .user_data_dir(profile_dir); + if !accept_invalid_certs { + builder = builder.respect_https_errors(); + } + let config = builder .build() .map_err(|error| format!("failed to build browser config: {error}"))?; diff --git a/crates/trusted-server-cli/src/commands/audit/collector.rs b/crates/trusted-server-cli/src/commands/audit/collector.rs index aca6814ca..ff7a45867 100644 --- a/crates/trusted-server-cli/src/commands/audit/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/collector.rs @@ -28,6 +28,15 @@ pub struct BrowserOpts { /// Hard cap in milliseconds on waiting for the page to settle. #[arg(long, default_value_t = 10_000)] pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as verification evidence, so an invalid + /// certificate could mean an impersonator is harvesting the session and + /// fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, } /// A request to collect a single page. diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index b1446933c..077c5550a 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -63,10 +63,15 @@ async fn collect_page_via_browser_async( "failed to create temporary browser profile for audit: {error}" )) })?; + // chromiumoxide ignores TLS errors by default. `generate` sends operator + // cookies and writes what it scrapes into the operator's config, so a + // certificate-invalid impersonator could both harvest the session and seed + // the config with slots of its choosing. Validate certificates. let config = BrowserConfig::builder() .chrome_executable(chrome_executable) .user_data_dir(user_data_dir.path()) .new_headless_mode() + .respect_https_errors() .build() .map_err(|error| { report_error(format!( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 43a8e7f8f..1feec9329 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -10,7 +10,9 @@ use std::io::Write; use std::path::{Path, PathBuf}; use serde::Serialize; -use trusted_server_core::creative_opportunities::CreativeOpportunitiesConfig; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, compile_page_pattern, +}; use url::Url; use crate::commands::audit::generate::collector::AuditCollector; @@ -23,6 +25,45 @@ use crate::error::{CliResult, cli_error, report_error}; use analyzer::{analyze_collected_page, extract_gtm_container_id}; +/// Writes `contents` to `path` atomically: a same-directory temp file is +/// written and fsynced, then renamed over the target, then the directory entry +/// is fsynced. +/// +/// A plain `fs::write` truncates the destination before writing, so a full disk +/// or an interrupted run would leave an operator's `trusted-server.toml` empty +/// or half-written. `rename` within a directory is atomic, so a reader sees +/// either the old file or the complete new one. +/// +/// The target's existing permissions are carried onto the replacement, since +/// the temp file is created 0600 and the config may intentionally be broader. +/// +/// # Errors +/// +/// Returns the underlying I/O error when the temp file cannot be created, +/// written, synced, or renamed over `path`. +fn write_file_atomically(path: &Path, contents: &str) -> std::io::Result<()> { + let directory = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + + let mut temp = tempfile::Builder::new() + .prefix(".ts-audit-") + .tempfile_in(directory)?; + temp.write_all(contents.as_bytes())?; + temp.as_file().sync_all()?; + if let Ok(metadata) = fs::metadata(path) { + temp.as_file().set_permissions(metadata.permissions())?; + } + temp.persist(path).map_err(|error| error.error)?; + + // Best-effort durability for the rename itself. Opening a directory handle + // is not portable (Windows rejects it), and the content is already safely + // on disk either way, so a failure here is not worth failing the command. + let _ = fs::File::open(directory).and_then(|handle| handle.sync_all()); + Ok(()) +} + /// Arguments for `ts audit generate ` — bootstraps draft Trusted Server /// config and JavaScript asset audit files from a live page (issue #800). #[derive(Debug, clap::Args)] @@ -229,7 +270,7 @@ fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliRes let mut written_paths = Vec::new(); if let Some(path) = &plan.js_assets_path { - fs::write(path, &outputs.js_assets_toml).map_err(|error| { + write_file_atomically(path, &outputs.js_assets_toml).map_err(|error| { report_error(format!( "failed to write JS asset audit {}: {error}", path.display() @@ -238,7 +279,7 @@ fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliRes written_paths.push(path.display().to_string()); } if let Some(path) = &plan.config_path { - fs::write(path, &outputs.draft_config_toml).map_err(|error| { + write_file_atomically(path, &outputs.draft_config_toml).map_err(|error| { report_error(format!( "failed to write draft config {}: {error}", path.display() @@ -488,6 +529,11 @@ pub(crate) fn run_update_slots( } else { page_patterns.to_vec() }; + // Reject a pattern the runtime cannot compile before it reaches the file: + // a persisted invalid glob either fails the next config load or is silently + // dropped at pattern-compile time, leaving the slot matching fewer pages + // than the config claims. + validate_page_patterns(&run_patterns)?; let merged = merge_slots(existing_creative, &discovered, &run_patterns, replace); let network_id = resolve_network_id( @@ -503,7 +549,7 @@ pub(crate) fn run_update_slots( .map_err(|error| report_error(format!("failed to write preview: {error}")))?; return Ok(()); } - fs::write(config_path, &updated).map_err(|error| { + write_file_atomically(config_path, &updated).map_err(|error| { report_error(format!( "failed to write config {}: {error}", config_path.display() @@ -518,6 +564,30 @@ pub(crate) fn run_update_slots( ) .map_err(|error| report_error(format!("failed to write command output: {error}"))) } +/// Rejects any page pattern the runtime's glob compiler would not accept. +/// +/// Uses [`compile_page_pattern`] so the accepted set is exactly what +/// `CreativeOpportunitySlot::compile_patterns` accepts at startup, including the +/// `**`→`*` normalisation. All patterns are reported at once so an operator +/// passing several `--page-pattern` values fixes them in one pass. +/// +/// # Errors +/// +/// Returns a user-facing error listing every pattern that does not compile. +fn validate_page_patterns(patterns: &[String]) -> CliResult<()> { + let invalid: Vec = patterns + .iter() + .filter_map(|pattern| compile_page_pattern(pattern).err()) + .collect(); + if invalid.is_empty() { + return Ok(()); + } + cli_error(format!( + "refusing to write invalid page pattern(s): {}", + invalid.join("; ") + )) +} + /// The default page pattern for a scraped URL: its path, or `/` for the root. fn default_page_pattern(target_url: &Url) -> String { let path = target_url.path(); @@ -591,6 +661,19 @@ mod tests { } } + /// A collected page carrying one discoverable GPT slot, for `run_update_slots`. + fn collected_page_with_header_slot() -> CollectedPage { + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + collected + } + fn audit_args(url: &str) -> GenerateArgs { GenerateArgs { url: url.to_string(), @@ -955,6 +1038,118 @@ mod tests { ); } + #[test] + fn update_slots_rejects_invalid_page_pattern_without_touching_config() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + let error = run_update_slots( + "https://publisher.example/", + &config_path, + None, + &["[".to_string()], + false, + &[], + false, + &collector, + &mut out, + ) + .expect_err("should reject an invalid glob"); + + assert!( + format!("{error:?}").contains("page pattern '['"), + "error should name the offending pattern, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "a rejected pattern must leave the operator config untouched" + ); + } + + #[test] + fn update_slots_accepts_double_star_pattern_like_the_runtime() { + // `/20**` does not compile directly but the runtime normalises it to + // `/20*`; validation must accept exactly what the runtime accepts. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + "https://publisher.example/", + &config_path, + None, + &["/20**".to_string()], + false, + &[], + false, + &collector, + &mut out, + ) + .expect("should accept a runtime-normalisable pattern"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["page_patterns"][0].as_str(), + Some("/20**") + ); + } + + #[test] + fn update_slots_write_replaces_the_config_without_leaving_temp_files() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + "https://publisher.example/", + &config_path, + None, + &[], + false, + &[], + false, + &collector, + &mut out, + ) + .expect("should update slots"); + + let entries: Vec = fs::read_dir(temp.path()) + .expect("should read temp dir") + .map(|entry| { + entry + .expect("should read entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + assert_eq!( + entries, + ["trusted-server.toml"], + "the atomic write should leave no stray temp file behind" + ); + let written = fs::read_to_string(&config_path).expect("should read config"); + toml::from_str::(&written).expect("rewritten config is valid TOML"); + } + #[test] fn update_slots_dry_run_does_not_persist_environment_overlay_config() { let temp = TempDir::new().expect("should create temp dir"); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 63f81b50c..580fe39ad 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -345,11 +345,26 @@ pub(super) fn splice_creative_slots( let rendered = rendered_slots.trim_matches('\n'); let existing = remove_inline_slot_value(existing)?; - // No section yet — append a fresh one with the network id and slots. - if !existing + // Presence is decided structurally (toml_edit), but the splice below is a + // line edit that only recognises a canonical `[creative_opportunities]` + // header. Reconciling the two here keeps a valid-but-unrecognised form — + // a quoted `["creative_opportunities"]` header, a top-level + // `creative_opportunities = { ... }` inline table, or a section implied + // only by its subtables — from being treated as absent and getting a + // duplicate table appended, which would produce invalid TOML. + let has_canonical_header = existing .lines() - .any(|line| is_table_header(line, "[creative_opportunities]")) - { + .any(|line| is_table_header(line, "[creative_opportunities]")); + if section_is_present(&existing)? && !has_canonical_header { + return cli_error( + "target config declares `creative_opportunities` in a form this updater cannot \ + edit safely; rewrite it as a `[creative_opportunities]` table (with \ + `[[creative_opportunities.slot]]` entries) and re-run", + ); + } + + // No section yet — append a fresh one with the network id and slots. + if !has_canonical_header { let mut result = existing; if !result.is_empty() && !result.ends_with('\n') { result.push('\n'); @@ -445,6 +460,22 @@ pub(super) fn splice_creative_slots( Ok(result) } +/// Whether `document` declares `creative_opportunities` at all, in any valid +/// TOML representation (canonical table, quoted header, inline table, or a +/// section implied only by its subtables). +/// +/// # Errors +/// +/// Returns an error when the document does not parse as TOML. +fn section_is_present(document: &str) -> CliResult { + let parsed = document.parse::().map_err(|error| { + report_error(format!( + "failed to parse target config before updating slots: {error}" + )) + })?; + Ok(parsed.get("creative_opportunities").is_some()) +} + /// Removes a scalar `creative_opportunities.slot` value so it can be replaced /// with the generated array-of-tables representation. fn remove_inline_slot_value(document: &str) -> CliResult { @@ -637,6 +668,108 @@ mod tests { toml::from_str::(&out).expect("spliced config is valid TOML"); } + #[test] + fn splice_rejects_quoted_section_header_instead_of_duplicating_it() { + // A quoted header is valid TOML but the line-based splice does not + // recognise it; appending a second `[creative_opportunities]` would + // produce a document that no longer parses. + let existing = "[\"creative_opportunities\"]\ngam_network_id = \"111\"\n"; + + let error = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect_err("should refuse an unrecognised section form"); + + assert!( + format!("{error:?}").contains("cannot edit safely"), + "error should tell the operator to rewrite the section, got {error:?}" + ); + } + + #[test] + fn splice_rejects_top_level_inline_creative_opportunities_table() { + let existing = "creative_opportunities = { gam_network_id = \"111\" }\n"; + + let error = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect_err("should refuse a top-level inline table"); + + assert!( + format!("{error:?}").contains("cannot edit safely"), + "error should tell the operator to rewrite the section, got {error:?}" + ); + } + + #[test] + fn splice_appends_section_when_config_has_none() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("appended config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222") + ); + } + + #[test] + fn splice_preserves_section_scalars_and_provider_subtables() { + // Mirrors the templated operator shape: section policy scalars in the + // head block and a per-slot prebid provider subtable. + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + auction_timeout_ms = 2000\n\ + section_root = \"homepage\"\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"ad-header-0\"\n\ + div_id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\ + [creative_opportunities.slot.providers.prebid]\n\ + bidders = {}\n\n\ + [auction]\nenabled = true\n"; + let existing_config = existing_config( + &existing + .replace("[creative_opportunities]\n", "") + .replace("[[creative_opportunities.slot]]", "[[slot]]") + .replace("[creative_opportunities.slot.", "[slot.") + .replace("\n[auction]\nenabled = true\n", ""), + ); + let discovered = discovered_header_slot(); + let merged = merge_slots( + Some(&existing_config), + &discovered, + &["/news/*".to_string()], + false, + ); + + let out = splice_creative_slots(existing, Some("111"), &render_slots(&merged)) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "section policy scalars must survive the splice" + ); + assert_eq!(creative["auction_timeout_ms"].as_integer(), Some(2000)); + assert_eq!( + creative["slot"][0]["gam_unit_path"].as_str(), + Some("/{network_id}/example/{section}"), + "an existing templated unit path must not be rewritten to a literal" + ); + assert!( + creative["slot"][0]["providers"]["prebid"]["bidders"].is_table(), + "the prebid provider subtable must be re-emitted" + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "trailing sections must be preserved" + ); + } + #[test] fn splice_preserves_crlf_line_endings() { let existing = "[creative_opportunities]\r\ngam_network_id = \"111\"\r\n\r\n\ diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 1a33b890a..e024eb8b1 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -156,6 +156,13 @@ pub(crate) struct AuditAdTemplatesVerifyArgs { /// Perform a deterministic scroll pass after the initial settle. #[arg(long)] pub scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + /// + /// Off by default: slots are matched on the post-redirect path, so an + /// off-origin page could otherwise satisfy `--strict`. Enable only for a + /// known redirect between your own properties (e.g. apex to `www`). + #[arg(long)] + pub allow_cross_origin_redirect: bool, /// Cookie to send with each page request, as `name=value`. Repeatable. /// Use to carry an existing session (e.g. a valid bot-protection clearance /// cookie) so the origin serves the real page instead of a challenge. diff --git a/crates/trusted-server-cli/src/commands/audit/page.rs b/crates/trusted-server-cli/src/commands/audit/page.rs index af0144fe9..6df54032d 100644 --- a/crates/trusted-server-cli/src/commands/audit/page.rs +++ b/crates/trusted-server-cli/src/commands/audit/page.rs @@ -4,6 +4,7 @@ use std::io::{self, Write}; use clap::Args; +use crate::ad_templates::output::escape_terminal_text; use crate::commands::audit::browser::BrowserCollector; use crate::commands::audit::collector::{ AuditCollector, BrowserCollectRequest, BrowserOpts, CollectedPage, @@ -57,11 +58,19 @@ fn write_summary(out: &mut dyn Write, url: &url::Url, page: &CollectedPage) -> R let to_err = |error: io::Error| format!("failed to write command output: {error}"); writeln!(out, "url: {url}").map_err(to_err)?; writeln!(out, "final url: {}", page.final_url).map_err(to_err)?; - writeln!(out, "title: {}", page.title).map_err(to_err)?; + // The title and collector warning messages are page-controlled, so escape + // control characters before they reach the operator's terminal. + writeln!(out, "title: {}", escape_terminal_text(&page.title)).map_err(to_err)?; writeln!(out, "scripts: {}", page.script_count).map_err(to_err)?; writeln!(out, "resources: {}", page.resource_count).map_err(to_err)?; for warning in &page.warnings { - writeln!(out, "warning [{}]: {}", warning.code, warning.message).map_err(to_err)?; + writeln!( + out, + "warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(to_err)?; } Ok(()) } diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs index 60deda068..4216db382 100644 --- a/crates/trusted-server-cli/src/commands/config/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -195,7 +195,14 @@ fn run_match(args: &AdTemplatesMatchArgs, out: &mut dyn Write) -> Result<(), Str }; let matched = match_slots(&config.slot, &path); - write_match_result(out, &path, &matched, &config.gam_network_id, args.details) + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + args.details, + ) } fn run_check(args: &AdTemplatesCheckArgs, out: &mut dyn Write) -> Result<(), String> { @@ -258,7 +265,14 @@ fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), }; let matched = match_slots(&config.slot, &path); - write_match_result(out, &path, &matched, &config.gam_network_id, true)?; + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + true, + )?; let method_pass = args.method.eq_ignore_ascii_case("GET"); let navigation_pass = !args.non_navigation; @@ -314,6 +328,7 @@ fn write_match_result( path: &str, matched: &[&CreativeOpportunitySlot], gam_network_id: &str, + section: &str, details: bool, ) -> Result<(), String> { if matched.is_empty() { @@ -330,7 +345,8 @@ fn write_match_result( if details { for slot in matched { - writeln!(out, "- {}", format_slot(slot, gam_network_id)).map_err(output_error)?; + writeln!(out, "- {}", format_slot(slot, gam_network_id, section)) + .map_err(output_error)?; } } @@ -341,7 +357,11 @@ fn write_gate(out: &mut dyn Write, label: &str, pass: bool) -> Result<(), String writeln!(out, "gate {label}: {}", if pass { "pass" } else { "block" }).map_err(output_error) } -fn format_slot(slot: &CreativeOpportunitySlot, gam_network_id: &str) -> String { +/// Formats one matched slot for `--details` output. +/// +/// `section` is the value the runtime derives from the evaluated path, so a +/// `{section}` template renders the same unit path the live request would use. +fn format_slot(slot: &CreativeOpportunitySlot, gam_network_id: &str, section: &str) -> String { let formats = slot .formats .iter() @@ -349,11 +369,16 @@ fn format_slot(slot: &CreativeOpportunitySlot, gam_network_id: &str) -> String { .collect::>() .join(", "); let providers = format_providers(slot); + // `None` means a dynamic template renders past GAM's unit-path byte limit — + // a config the runtime rejects, so surface it rather than printing a path. + let gam_unit_path = slot + .render_gam_unit_path(gam_network_id, section) + .unwrap_or_else(|| "".to_string()); format!( "{} div={} gam={} patterns=[{}] formats=[{}] providers=[{}]", slot.id, slot.resolved_div_id(), - slot.resolved_gam_unit_path(gam_network_id), + gam_unit_path, slot.page_patterns.join(", "), formats, providers, diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 594dbb6f9..f63f92d8e 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -531,15 +531,7 @@ impl CreativeOpportunitySlot { // skip `compile_patterns`). Re-compiles on every call. self.page_patterns .iter() - .any(|pattern| match Pattern::new(pattern) { - Ok(p) => p.matches(path), - Err(_) => { - let normalised = pattern.replace("**", "*"); - Pattern::new(&normalised) - .map(|p| p.matches(path)) - .unwrap_or(false) - } - }) + .any(|pattern| compile_page_pattern(pattern).is_ok_and(|p| p.matches(path))) } /// Compile [`page_patterns`](Self::page_patterns) into the @@ -556,22 +548,20 @@ impl CreativeOpportunitySlot { self.compiled_patterns = self .page_patterns .iter() - .filter_map(|pattern| { - match Pattern::new(pattern).or_else(|_| Pattern::new(&pattern.replace("**", "*"))) { - Ok(compiled) => Some(compiled), - Err(_) => { - // Build-time validation only requires *one* valid pattern - // per slot, so a mixed valid/invalid set passes the build - // with the bad pattern silently dropped here. Warn so the - // operator can see the slot matches fewer pages than - // configured. - log::warn!( - "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", - self.id, - pattern - ); - None - } + .filter_map(|pattern| match compile_page_pattern(pattern) { + Ok(compiled) => Some(compiled), + Err(_) => { + // Build-time validation only requires *one* valid pattern + // per slot, so a mixed valid/invalid set passes the build + // with the bad pattern silently dropped here. Warn so the + // operator can see the slot matches fewer pages than + // configured. + log::warn!( + "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", + self.id, + pattern + ); + None } }) .collect(); @@ -834,6 +824,37 @@ pub struct PrebidSlotParams { pub bidders: HashMap, } +/// Compiles a [`page_patterns`](CreativeOpportunitySlot::page_patterns) entry +/// using the runtime's normalisation. +/// +/// This is the single definition of what the runtime accepts as a page glob: +/// a direct [`Pattern::new`], falling back to the `**`→`*` rewrite that +/// [`CreativeOpportunitySlot::compile_patterns`] and +/// [`matches_path`](CreativeOpportunitySlot::matches_path) apply. Tooling that +/// writes patterns into operator config validates them through this function so +/// it cannot persist a pattern the runtime would silently drop. +/// +/// # Errors +/// +/// Returns an error string when the pattern compiles neither directly nor after +/// normalisation. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::creative_opportunities::compile_page_pattern; +/// +/// assert!(compile_page_pattern("/news/*").is_ok()); +/// // `**` in a position the glob crate rejects is normalised to `*`. +/// assert!(compile_page_pattern("/20**").is_ok()); +/// assert!(compile_page_pattern("[").is_err()); +/// ``` +pub fn compile_page_pattern(pattern: &str) -> Result { + Pattern::new(pattern) + .or_else(|_| Pattern::new(&pattern.replace("**", "*"))) + .map_err(|error| format!("page pattern '{pattern}' is not a valid glob: {error}")) +} + /// Validates that a slot ID contains only safe characters. /// /// Allowed characters: ASCII alphanumerics, underscores (`_`), and hyphens (`-`). diff --git a/docs/guide/cli.md b/docs/guide/cli.md index bd1157937..e0baac367 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -138,6 +138,27 @@ ts audit generate https://publisher.example --force The legacy `ts audit ` form remains a compatibility alias for artifact generation. New automation should use `ts audit generate `. +### Audit safety defaults + +Every `ts audit` browser session validates TLS certificates. This matters +because `--cookie` sends a real session to the origin and the page's own +response becomes the audit's evidence, so a certificate-invalid host could both +harvest the session and fabricate what the audit reports. Override only for a +host you control with a known self-signed certificate: + +```bash +ts audit page https://staging.publisher.example --danger-accept-invalid-certs +``` + +`ts audit ad-templates verify` matches configured slots against the +**post-redirect** path, so it refuses a redirect that leaves the requested +origin rather than accepting another site's evidence as verification. Allow it +for a known redirect between your own properties (for example apex to `www`): + +```bash +ts audit ad-templates verify https://publisher.example/ --allow-cross-origin-redirect +``` + `ts audit` is not an EdgeZero adapter command. It has no `--adapter` option and it does not provision resources, push config, build, deploy, or contact platform APIs. From 07b37a1f0a2a4436437b57bf5b634dd3cf330aa7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 10:47:37 +0530 Subject: [PATCH 255/395] Verify generated ad-template config before it replaces the operator file `ts audit ad-templates generate` derived everything it wrote from a live, page-controlled ad stack and never checked the result, so several reachable inputs produced a config that cannot load. An unloadable `trusted-server.toml` is not a degraded ad stack: `build_state` fails and the adapter answers every route from the startup error router, so the whole site returns 500 once pushed. Add a write-side gate that runs the candidate through `Settings::from_toml`, the same `finalize_deserialized` chain the runtime uses at startup. It runs on the `--dry-run` path too, so a clean preview is now evidence the config loads. When the target config was already unloadable before the run, the gate reports that as a warning instead of blaming this run, so a freshly bootstrapped file carrying placeholder secrets can still be updated. Close the three reachable paths at their source as well: - Skip a scraped slot whose ad-unit path contains `{` or `}`. The path is a template and there is no escape syntax, so a literal brace either fails config load or is silently reinterpreted as a placeholder. - Skip a slot whose div id normalizes to nothing (a wholly ephemeral id such as a React SSR marker). An empty `div_id` fails config load, and as a runtime prefix it would bind the slot to the first id-bearing element on the page. - Refuse to create a `[creative_opportunities]` section with no GAM network id rather than writing one that omits the required key. This is reachable because the network id is only recovered from an all-digit leading segment, which an MCM child-network path does not have. --- .../src/commands/audit/generate/gpt_slots.rs | 89 ++++++++++++++ .../src/commands/audit/generate/mod.rs | 72 +++++++++++ .../src/commands/audit/generate/slot_toml.rs | 34 +++++- .../src/commands/audit/generate/validate.rs | 112 ++++++++++++++++++ 4 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/validate.rs diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index fea34dd1f..365a5b696 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -130,6 +130,9 @@ fn slot_from_registry(entry: &CollectedGptSlot, page_has_prebid: bool) -> Option if is_multi_slot_div(&entry.div_id) { return None; } + if !is_usable_unit_path(&entry.gam_unit_path) { + return None; + } let formats: Vec<(u32, u32)> = entry .sizes .iter() @@ -140,6 +143,14 @@ fn slot_from_registry(entry: &CollectedGptSlot, page_has_prebid: bool) -> Option return None; } let div_stem = normalize_div_stem(&entry.div_id); + // Normalization truncates at the first ephemeral marker, so a div id that is + // *entirely* ephemeral (`_R_9sl…`, or exactly `-container`) reduces to the + // empty string. An empty `div_id` override fails config load outright, and + // an empty prefix would bind the slot to the first id-bearing element on the + // page, so such a slot is unusable rather than merely imprecise. + if div_stem.is_empty() { + return None; + } Some(DiscoveredSlot { id: slot_id_from_div(&div_stem), div_id: div_stem, @@ -155,6 +166,17 @@ fn is_multi_slot_div(div_id: &str) -> bool { div_id.contains('~') } +/// Whether a scraped GAM ad-unit path can be represented in config. +/// +/// `gam_unit_path` is a template: `{` and `}` delimit placeholders and +/// [`parse_unit_template`](trusted_server_core::creative_opportunities) offers no +/// escape syntax. A live path containing a brace would either fail config load +/// or, worse, be silently reinterpreted as a placeholder-bearing template. A +/// blank path is rejected for the same reason config load rejects it. +fn is_usable_unit_path(path: &str) -> bool { + !path.trim().is_empty() && !path.contains(['{', '}']) +} + /// Strips ephemeral GPT div-id noise so the stored id is stable across renders. /// /// Removes a trailing `-container` wrapper, then truncates at the first ephemeral @@ -220,6 +242,9 @@ fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot)> { .filter(|segment| segment.bytes().all(|byte| byte.is_ascii_digit()))? .to_string(); let gam_unit_path = format!("/{}", iu_parts.replace(',', "/")); + if !is_usable_unit_path(&gam_unit_path) { + return None; + } // A usable unit path needs the network id plus at least one path segment. parts.next()?; @@ -232,6 +257,11 @@ fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot)> { return None; } let div_id = normalize_div_stem(&raw_div); + // See `slot_from_registry`: a fully ephemeral div id normalizes to nothing, + // which is neither a valid config value nor a usable runtime prefix. + if div_id.is_empty() { + return None; + } let formats = parse_sizes(sizes_raw.as_deref().or(fallback_sizes_raw.as_deref())?); if formats.is_empty() { @@ -466,6 +496,65 @@ mod tests { } } + #[test] + fn registry_slot_with_brace_in_unit_path_is_skipped() { + // `gam_unit_path` is a template and there is no escape syntax, so a + // literal brace either fails config load or is silently reinterpreted as + // a placeholder. Neither is acceptable to persist. + let registry = vec![ + registry_slot("/123/home/{section}", "div-gpt-ad-a", &[(300, 250)]), + registry_slot("/123/home/ok", "div-gpt-ad-b", &[(300, 250)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "the brace-bearing slot should be dropped, the clean one kept" + ); + assert_eq!(discovered.slots[0].gam_unit_path, "/123/home/ok"); + } + + #[test] + fn registry_slot_whose_div_id_is_entirely_ephemeral_is_skipped() { + // `_R_…` is a React SSR marker; normalizing truncates at it, leaving an + // empty stem. An empty div_id fails config load, and as a runtime prefix + // it would match the first id-bearing element on the page. + let registry = vec![registry_slot( + "/123/home/header", + "_R_9slkta7pd6", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a slot with no stable div stem should be dropped, got {:?}", + discovered.slots + ); + } + + #[test] + fn volatile_guid_div_id_still_normalizes_to_a_usable_prefix() { + // The live autoblog shape: a GUID between two copies of the slot name. + // This must survive - only a stem that normalizes to *nothing* is dropped. + let registry = vec![registry_slot( + "/88059007/autoblog/homepage", + "ad-in_content-0949b6c5726343bf8bbec2ac47b494b4-in_content-0", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!( + discovered.slots[0].div_id, "ad-in_content", + "the GUID and trailing index should be truncated to a stable prefix" + ); + } + #[test] fn reads_slots_from_live_registry() { let registry = vec![registry_slot( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 1feec9329..d3125a1f5 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod browser_collector; pub(crate) mod collector; mod gpt_slots; mod slot_toml; +mod validate; use std::collections::BTreeSet; use std::fs; @@ -544,6 +545,15 @@ pub(crate) fn run_update_slots( let rendered_slots = render_slots(&merged); let updated = splice_creative_slots(&existing, network_id.as_deref(), &rendered_slots)?; + // Everything above is derived from a live, page-controlled ad stack, so the + // candidate has to clear the runtime's own load path before it can replace + // the operator's file. This runs on the dry-run path too — otherwise "the + // preview looked fine" would not be evidence that the config loads. + for warning in validate::check_candidate(&updated, &existing)? { + writeln!(out, "warning: {warning}") + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + if dry_run { writeln!(out, "{updated}") .map_err(|error| report_error(format!("failed to write preview: {error}")))?; @@ -1150,6 +1160,68 @@ mod tests { toml::from_str::(&written).expect("rewritten config is valid TOML"); } + /// A full, loadable config with real secrets substituted, so the write-side + /// validation gate is live rather than downgraded by a broken baseline. + fn loadable_config() -> String { + EXAMPLE_CONFIG + .replace( + "replace-with-admin-password-32-bytes", + "test-admin-password-32-bytes-minimum", + ) + .replace( + "trusted-server-placeholder-secret", + "test-ec-passphrase-32-bytes-minimum", + ) + .replace( + "change-me-proxy-secret", + "test-proxy-secret-32-bytes-minimum", + ) + } + + #[test] + fn generated_config_loads_through_the_runtime_settings_path() { + // The end-to-end contract: whatever `generate` writes must survive the + // same load path the adapter runs at startup. An unloadable config is a + // full-site outage once pushed, not a degraded ad stack. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let baseline = loadable_config(); + trusted_server_core::settings::Settings::from_toml(&baseline) + .expect("test baseline must itself be loadable or the gate is not exercised"); + fs::write(&config_path, &baseline).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + "https://publisher.example/", + &config_path, + None, + &[], + false, + &[], + false, + &collector, + &mut out, + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let settings = trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + let creative = settings + .creative_opportunities + .expect("generated config should carry creative opportunities"); + assert_eq!( + creative.slot.len(), + 1, + "the discovered slot should be present after a real load" + ); + assert_eq!( + creative.slot[0].div_id.as_deref(), + Some("div-gpt-ad-header") + ); + } + #[test] fn update_slots_dry_run_does_not_persist_environment_overlay_config() { let temp = TempDir::new().expect("should create temp dir"); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 580fe39ad..880ce1cac 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -365,14 +365,25 @@ pub(super) fn splice_creative_slots( // No section yet — append a fresh one with the network id and slots. if !has_canonical_header { + // `gam_network_id` is a required field, so creating the section without + // one writes a config that cannot load at all. This is reachable: the + // network id is only recovered when the scraped unit path starts with an + // all-digit segment, which an MCM/child-network path like + // `/1234,5678/home/header` does not. + let Some(network_id) = network_id else { + return cli_error( + "refusing to create a `[creative_opportunities]` section without a \ + GAM network id: none could be determined from the audited page, and \ + the key is required. Add `[creative_opportunities]` with a \ + `gam_network_id` to the config and re-run", + ); + }; let mut result = existing; if !result.is_empty() && !result.ends_with('\n') { result.push('\n'); } result.push_str("\n[creative_opportunities]\n"); - if let Some(network_id) = network_id { - result.push_str(&format!("gam_network_id = {}\n", toml_string(network_id))); - } + result.push_str(&format!("gam_network_id = {}\n", toml_string(network_id))); result.push_str(rendered); result.push('\n'); return Ok(result); @@ -697,6 +708,23 @@ mod tests { ); } + #[test] + fn splice_refuses_fresh_section_without_a_network_id() { + // Reachable whenever the scraped unit path has no all-digit leading + // segment (MCM/child-network paths). Writing the section anyway produces + // a config missing a required field, which fails load and takes every + // route to the startup error router once pushed. + let existing = "[publisher]\ndomain = \"x\"\n"; + + let error = splice_creative_slots(existing, None, &header_rendered()) + .expect_err("should refuse to create a section with no network id"); + + assert!( + format!("{error:?}").contains("without a GAM network id"), + "error should name the missing network id, got {error:?}" + ); + } + #[test] fn splice_appends_section_when_config_has_none() { let existing = "[publisher]\ndomain = \"x\"\n"; diff --git a/crates/trusted-server-cli/src/commands/audit/generate/validate.rs b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs new file mode 100644 index 000000000..721ba4046 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs @@ -0,0 +1,112 @@ +//! Write-side validation for generated ad-template config. +//! +//! Everything the generator writes is derived from a live, page-controlled ad +//! stack, so the candidate document has to clear the same bar the runtime +//! applies at startup *before* it replaces the operator's file. A config the +//! runtime rejects is not a degraded ad stack — `build_state` fails and the +//! adapter answers every route from the startup error router, so an unloadable +//! `trusted-server.toml` is a full-site outage once pushed. + +use trusted_server_core::settings::Settings; + +use crate::error::{CliResult, cli_error}; + +/// Validates the candidate config text the generator is about to persist. +/// +/// Runs [`Settings::from_toml`], which drives the identical +/// `finalize_deserialized` chain the runtime uses — serde (`deny_unknown_fields` +/// plus required fields), then `compile_slots` → `compile_unit_templates` → +/// `validate_runtime`, then the validator pass — with no I/O. +/// +/// `baseline` is the config as it was read from disk. When the baseline is +/// *already* unloadable, this run cannot be blamed for it: the candidate is +/// accepted and the pre-existing error is returned as a warning instead. Without +/// that escape hatch a freshly bootstrapped config carrying placeholder secrets +/// could never be updated by `generate`. +/// +/// # Errors +/// +/// Returns a user-facing error when the candidate fails to load and the baseline +/// loaded cleanly — that is, when this run introduced the failure. +pub(super) fn check_candidate(candidate: &str, baseline: &str) -> CliResult> { + let Err(candidate_error) = Settings::from_toml(candidate) else { + return Ok(Vec::new()); + }; + + if let Err(baseline_error) = Settings::from_toml(baseline) { + return Ok(vec![format!( + "target config was already invalid before this run, so the generated \ + result could not be verified: {baseline_error}" + )]); + } + + cli_error(format!( + "refusing to write: the generated config would fail to load, which would \ + take the service down once pushed: {candidate_error}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A minimal config that loads cleanly, used as the valid baseline. + fn baseline() -> String { + crate::commands::config::init::EXAMPLE_CONFIG + .replace( + "replace-with-admin-password-32-bytes", + "test-admin-password-32-bytes-minimum", + ) + .to_string() + } + + #[test] + fn valid_candidate_passes_without_warnings() { + let config = baseline(); + + let warnings = check_candidate(&config, &config).expect("valid candidate should pass"); + + assert!( + warnings.is_empty(), + "a clean candidate should not warn, got {warnings:?}" + ); + } + + #[test] + fn candidate_this_run_broke_is_refused() { + let good = baseline(); + // An empty div_id override is exactly what a div id normalized down to + // nothing would produce, and `validate_runtime` rejects it. + let broken = format!( + "{good}\n[[creative_opportunities.slot]]\n\ + id = \"broken\"\ndiv_id = \"\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 300, height = 250 }}]\n" + ); + + let error = check_candidate(&broken, &good).expect_err("should refuse a broken candidate"); + + assert!( + format!("{error:?}").contains("refusing to write"), + "error should name the refusal, got {error:?}" + ); + } + + #[test] + fn pre_existing_breakage_downgrades_to_a_warning() { + // The operator's file was already unloadable; `generate` must still be + // able to update it rather than blaming this run for the old error. + let broken_baseline = "[creative_opportunities]\n"; + let broken_candidate = "[creative_opportunities]\n"; + + let warnings = check_candidate(broken_candidate, broken_baseline) + .expect("a pre-existing failure should not block the write"); + + assert_eq!(warnings.len(), 1, "should surface exactly one warning"); + assert!( + warnings[0].contains("already invalid"), + "warning should name the pre-existing failure, got {:?}", + warnings[0] + ); + } +} From e7f8268743e847bdd4e59df4a8285390fd3e866d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 10:55:22 +0530 Subject: [PATCH 256/395] Add multi-page collection and crawl planning for ad-template generate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groundwork for discovering ad slots across a site's sections rather than from a single page. Nothing calls this yet; `run_update_slots` is unchanged. `AuditCollector` gains a defaulted `collect_pages` that streams each page to a sink, so every existing implementor keeps working and the caller can fold a page into its evidence and drop the DOM immediately instead of holding every serialization at once. The browser collector overrides it to launch Chrome once for the whole crawl: a cold start plus a fresh profile dominates the cost of a multi-page run, and the shared profile carries a bot-protection clearance cookie earned on the first page across the rest of the walk. Page discovery reads the hydrated DOM rather than the served markup, because an app-router page keeps its link graph in the framework payload — parsing raw HTML finds only a fraction of a site's sections. Sitemaps are fetched from inside the open page via `fetch` plus `DOMParser`, which inherits the session's cookies and Chrome's TLS fingerprint, gets transparent gzip and XML parsing, and so needs no new Rust dependency. `crawl_plan` turns links and sitemap entries into a bounded page set: one landing page and one article per section, ranked by whether navigation and the sitemap corroborate each other, capped by section and page budgets. Sections dropped for budget are reported rather than silently omitted. Same-origin is enforced on links and on sitemap entries alike, since a `Sitemap:` directive can name any host and the crawl carries operator cookies. --- .../src/commands/audit/generate/analyzer.rs | 12 + .../audit/generate/browser_collector.rs | 217 +++++++- .../src/commands/audit/generate/collector.rs | 78 +++ .../src/commands/audit/generate/crawl_plan.rs | 521 ++++++++++++++++++ .../src/commands/audit/generate/mod.rs | 5 + 5 files changed, 823 insertions(+), 10 deletions(-) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs diff --git a/crates/trusted-server-cli/src/commands/audit/generate/analyzer.rs b/crates/trusted-server-cli/src/commands/audit/generate/analyzer.rs index dc1ea9ffe..06d784b7a 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/analyzer.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/analyzer.rs @@ -286,6 +286,8 @@ mod tests { resource_type: Some("Script".to_string()), }], gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: vec!["partial settle".to_string()], }; @@ -323,6 +325,8 @@ mod tests { script_tags: Vec::new(), network_requests: Vec::new(), gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; @@ -341,6 +345,8 @@ mod tests { script_tags: Vec::new(), network_requests: Vec::new(), gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; @@ -366,6 +372,8 @@ mod tests { resource_type: Some("script".to_string()), }], gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; @@ -401,6 +409,8 @@ mod tests { ], network_requests: Vec::new(), gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; @@ -432,6 +442,8 @@ mod tests { script_tags: Vec::new(), network_requests: Vec::new(), gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index 077c5550a..b1c504cd5 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -13,7 +13,8 @@ use url::Url; use which::which; use crate::commands::audit::generate::collector::{ - AuditCollector, CollectedGptSlot, CollectedPage, CollectedRequest, CollectedScriptTag, + AuditCollector, CollectedGptSlot, CollectedLink, CollectedPage, CollectedRequest, + CollectedScriptTag, ControlFlow, PageSink, }; use crate::error::{CliResult, report_error}; @@ -49,14 +50,56 @@ impl AuditCollector for BrowserAuditCollector { )) })?; - runtime.block_on(collect_page_via_browser_async(target_url, cookies)) + runtime.block_on(async { + let mut collected = None; + with_browser( + std::slice::from_ref(target_url), + cookies, + &mut |_, result| { + collected = Some(result); + Ok(ControlFlow::Stop) + }, + ) + .await?; + collected.unwrap_or_else(|| Err(report_error("browser session produced no page"))) + }) + } + + fn collect_pages( + &self, + targets: &[Url], + cookies: &[(String, String)], + on_page: PageSink<'_>, + ) -> CliResult<()> { + if targets.is_empty() { + return Ok(()); + } + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| { + report_error(format!( + "failed to build Tokio runtime for browser audit: {error}" + )) + })?; + + runtime.block_on(with_browser(targets, cookies, on_page)) } } -async fn collect_page_via_browser_async( - target_url: &Url, +/// Launches one browser, walks `targets` on it, and hands each result to `sink`. +/// +/// One launch for the whole crawl rather than one per page: a cold Chrome start +/// plus a fresh profile dominates the cost of a multi-page run. The shared +/// profile is also load-bearing — a bot-protection clearance cookie earned on +/// the first page carries to the rest of the crawl, which is what makes a +/// multi-section walk of a protected site viable at all. The tradeoff is that +/// paywall meters and personalization also accumulate across the run. +async fn with_browser( + targets: &[Url], cookies: &[(String, String)], -) -> CliResult { + sink: PageSink<'_>, +) -> CliResult<()> { let chrome_executable = find_browser_executable()?; let user_data_dir = TempDir::new().map_err(|error| { report_error(format!( @@ -93,13 +136,25 @@ async fn collect_page_via_browser_async( } }); - let result = collect_page_from_browser(&mut browser, target_url, cookies).await; + // Sitemap discovery is a whole-site fact, so only the first target pays for it. + let mut result = Ok(()); + for (index, target) in targets.iter().enumerate() { + let collected = collect_page_from_browser(&mut browser, target, cookies, index == 0).await; + match sink(target, collected) { + Ok(ControlFlow::Continue) => {} + Ok(ControlFlow::Stop) => break, + Err(error) => { + result = Err(error); + break; + } + } + } let close_result = timeout(BROWSER_CLOSE_TIMEOUT, browser.close()) .await .map_err(|_| report_error("timed out closing browser after audit")) - .and_then(|result| { - result.map_err(|error| { + .and_then(|closed| { + closed.map_err(|error| { report_error(format!("failed to close browser after audit: {error}")) }) }); @@ -109,15 +164,21 @@ async fn collect_page_via_browser_async( let _ = handler_task.await; match (result, close_result) { - (Ok(collected), Ok(_)) => Ok(collected), - (Ok(_), Err(error)) | (Err(error), _) => Err(error), + (Ok(()), Ok(_)) => Ok(()), + (Ok(()), Err(error)) | (Err(error), _) => Err(error), } } +/// Collects one page on an already-launched browser. +/// +/// `discover_sitemap` runs the `robots.txt`/sitemap fetch from inside this +/// page's context. It is meaningful only once per crawl (the site's sitemap does +/// not change per page), so callers pass `true` for the root page only. async fn collect_page_from_browser( browser: &mut Browser, target_url: &Url, cookies: &[(String, String)], + discover_sitemap: bool, ) -> CliResult { let page = browser.new_page("about:blank").await.map_err(|error| { report_error(format!("failed to create browser page for audit: {error}")) @@ -238,6 +299,34 @@ async fn collect_page_from_browser( Err(_) => Vec::new(), }; + // Links come from the hydrated DOM, not the served markup: an app-router + // page keeps its link graph in the framework payload, so parsing the raw + // HTML finds only a fraction of the site's sections. Best-effort — an empty + // list just means crawl planning falls back to other sources. + let links: Vec = match page.evaluate(LINKS_SCRIPT).await { + Ok(result) => result.into_value().unwrap_or_default(), + Err(_) => Vec::new(), + }; + + // Sitemap discovery is a whole-site fact, so it runs once per crawl. A miss + // is normal (no sitemap, robots 404, fetch blocked) and leaves planning to + // the link graph alone. + let mut sitemap_locs: Vec = if discover_sitemap { + match page.evaluate(SITEMAP_SCRIPT).await { + Ok(result) => result.into_value().unwrap_or_default(), + Err(_) => Vec::new(), + } + } else { + Vec::new() + }; + sitemap_locs.truncate(MAX_SITEMAP_LOCS); + if discover_sitemap && sitemap_locs.is_empty() { + warnings.push( + "no sitemap was reachable; site sections were inferred from page links only" + .to_string(), + ); + } + Ok(CollectedPage { requested_url: target_url.to_string(), final_url, @@ -258,10 +347,118 @@ async fn collect_page_from_browser( }) .collect(), gpt_slots, + links, + sitemap_locs, warnings, }) } +/// Maximum sitemap `` entries kept. Section discovery needs one page per +/// section, so a 50,000-URL catalog sitemap is truncated hard. +const MAX_SITEMAP_LOCS: usize = 5000; + +/// Reads same-origin `a[href]` targets from the hydrated DOM. +/// +/// `anchor.href` is absolutized by the DOM already, and `in_nav` records whether +/// the anchor sits inside site navigation — navigation is the publisher's own +/// declaration of its taxonomy, so those links rank higher when picking sections. +/// +/// Reading the DOM rather than the served markup is deliberate: an app-router +/// page keeps its link graph in the framework payload, so parsing raw HTML finds +/// only a fraction of a site's sections. +const LINKS_SCRIPT: &str = r#"() => { + try { + const navAnchors = new Set( + Array.from(document.querySelectorAll( + 'nav a[href], header a[href], [role="navigation"] a[href]' + )) + ); + const out = []; + const seen = new Set(); + for (const anchor of document.querySelectorAll('a[href]')) { + if (out.length >= 2000) break; + const href = anchor.href; + if (!href || seen.has(href)) continue; + if (!href.startsWith(location.origin)) continue; + seen.add(href); + out.push({ url: href, in_nav: navAnchors.has(anchor) }); + } + return out; + } catch (error) { + return []; + } +}"#; + +/// Discovers sitemap page URLs from inside the page, starting at `robots.txt`. +/// +/// Runs in the browser rather than through a Rust HTTP client on purpose: the +/// in-page `fetch` carries the session's cookies and Chrome's TLS fingerprint, +/// so a bot-protection layer that would answer a bare client with a challenge +/// serves the real document instead. It also gets transparent gzip and an XML +/// parser for free, which is why sitemap support needs no new Rust dependency. +/// +/// Same-origin is enforced here *and* again in Rust: a `Sitemap:` directive can +/// name any host, and this crawl carries operator-supplied cookies. +const SITEMAP_SCRIPT: &str = r#"async () => { + const sameOrigin = (raw) => { + try { + return new URL(raw, location.origin).origin === location.origin; + } catch (error) { + return false; + } + }; + const fetchText = async (url) => { + try { + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) return null; + return await response.text(); + } catch (error) { + return null; + } + }; + const parseLocs = (text) => { + try { + const doc = new DOMParser().parseFromString(text, 'application/xml'); + if (doc.querySelector('parsererror')) return { pages: [], indexes: [] }; + const indexes = Array.from(doc.querySelectorAll('sitemapindex > sitemap > loc')) + .map((node) => (node.textContent || '').trim()).filter(sameOrigin); + const pages = Array.from(doc.querySelectorAll('urlset > url > loc')) + .map((node) => (node.textContent || '').trim()).filter(sameOrigin); + return { pages, indexes }; + } catch (error) { + return { pages: [], indexes: [] }; + } + }; + + const roots = []; + const robots = await fetchText('/robots.txt'); + if (robots) { + for (const line of robots.split(/\r?\n/)) { + const match = /^\s*sitemap\s*:\s*(\S+)/i.exec(line); + if (match && sameOrigin(match[1])) roots.push(match[1]); + } + } + if (roots.length === 0) roots.push('/sitemap.xml', '/sitemap_index.xml'); + + const pages = []; + let childrenFollowed = 0; + for (const root of roots) { + if (pages.length >= 5000) break; + const text = await fetchText(root); + if (!text) continue; + const parsed = parseLocs(text); + pages.push(...parsed.pages); + for (const child of parsed.indexes) { + if (childrenFollowed >= 10 || pages.length >= 5000) break; + childrenFollowed += 1; + const childText = await fetchText(child); + if (!childText) continue; + pages.push(...parseLocs(childText).pages); + } + } + return pages.slice(0, 5000); +}"#; + /// Reads the live GPT slot registry into `{gam_unit_path, div_id, sizes}` rows. /// /// Mirrors the ad-template verifier's `getSlots()` scrape: it defends against a diff --git a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs index 2a31c763b..625ac660e 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs @@ -3,6 +3,26 @@ use url::Url; use crate::error::CliResult; +/// Sink invoked once per collected page during a batch crawl. +/// +/// Receives the per-page outcome so a failed page can be folded into the run as +/// a warning rather than aborting it; returning `Err` stops the crawl. +pub(crate) type PageSink<'a> = + &'a mut dyn FnMut(&Url, CliResult) -> CliResult; + +/// Whether a batch crawl should keep going after a page. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ControlFlow { + /// Collect the next target. + #[allow( + dead_code, + reason = "constructed by run_update_slots once it orchestrates the crawl" + )] + Continue, + /// Stop the crawl without an error (budget reached, challenge rate exceeded). + Stop, +} + pub(crate) trait AuditCollector { /// Collects a live page. `cookies` are `(name, value)` pairs set on the /// browser context before navigation (scoped to `target_url`) so an existing @@ -13,6 +33,41 @@ pub(crate) trait AuditCollector { target_url: &Url, cookies: &[(String, String)], ) -> CliResult; + + /// Collects several pages in one session, handing each result to `on_page`. + /// + /// The default implementation loops over [`collect_page`](Self::collect_page), + /// which keeps every existing implementor working unchanged. The browser + /// collector overrides it to reuse one Chrome instance and profile across the + /// crawl — a fresh launch per page dominates the cost of a multi-page run, + /// and a shared profile carries bot-protection clearance cookies site-wide. + /// + /// Results are streamed rather than returned as a `Vec` so the caller can + /// fold each page into its evidence and drop the page's HTML immediately, + /// instead of holding every DOM serialization at once. + /// + /// # Errors + /// + /// Returns an error when `on_page` does, or when the session itself cannot + /// be established. Individual page failures are delivered to `on_page`. + #[allow( + dead_code, + reason = "called by run_update_slots once it orchestrates the crawl" + )] + fn collect_pages( + &self, + targets: &[Url], + cookies: &[(String, String)], + on_page: PageSink<'_>, + ) -> CliResult<()> { + for target in targets { + let collected = self.collect_page(target, cookies); + if on_page(target, collected)? == ControlFlow::Stop { + break; + } + } + Ok(()) + } } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] @@ -29,9 +84,32 @@ pub(crate) struct CollectedPage { /// when the ad request never fires (consent-gated or iframe-issued). #[serde(default)] pub(crate) gpt_slots: Vec, + /// Same-origin `a[href]` targets read from the hydrated DOM, absolutized. + /// + /// Read from the live DOM rather than the served HTML on purpose: an + /// app-router page keeps its link graph in the framework payload, so parsing + /// the raw markup finds only a fraction of the site's sections. + #[serde(default)] + pub(crate) links: Vec, + /// Sitemap `` entries discovered from `robots.txt`, when fetched. + /// + /// Empty unless sitemap discovery ran (root page only). + #[serde(default)] + pub(crate) sitemap_locs: Vec, pub(crate) warnings: Vec, } +/// A same-origin link observed in the hydrated DOM. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedLink { + /// Absolute URL of the link target. + pub(crate) url: String, + /// Whether the anchor sits inside site navigation (`nav`, `header`, + /// `[role="navigation"]`). Nav links are the publisher's own declaration of + /// its taxonomy, so they rank above body links when choosing sections. + pub(crate) in_nav: bool, +} + /// A single slot read from the page's live GPT registry. #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] pub(crate) struct CollectedGptSlot { diff --git a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs new file mode 100644 index 000000000..5b959dc4f --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs @@ -0,0 +1,521 @@ +//! Pure crawl planning: turn discovered links and sitemap entries into the +//! bounded set of pages worth loading in a browser. +//! +//! The goal is deliberately *not* site coverage. Ad slots repeat per site +//! section, and the generated config needs one glob pair per section +//! (`/news` and `/news/*`), so one representative page per section is enough. +//! That keeps the crawl proportional to the publisher's taxonomy (a dozen +//! sections) rather than its catalog (tens of thousands of articles). +//! +//! Two sources feed the plan and each supplies a half the other cannot: +//! +//! - **Navigation links** give section *landing* paths (`/news`), which +//! sitemaps routinely omit, and are the publisher's own taxonomy declaration. +//! - **Sitemap entries** give a real *article* per section (`/news/story-abc`), +//! which is where in-content slots live, and reveal sections hidden behind a +//! navigation overflow menu. +#![allow( + dead_code, + reason = "planner is exercised by tests until run_update_slots orchestrates the crawl" +)] + +use std::collections::BTreeMap; + +use url::Url; + +use super::collector::CollectedLink; + +/// Path segments that are never a content section worth sampling. +/// +/// These carry either no ad stack at all or an unrepresentative one, and +/// crawling them spends budget that a real section needs. +const NOISE_SEGMENTS: &[&str] = &[ + "about", + "about-us", + "account", + "author", + "cart", + "contact", + "editorial-policy", + "login", + "logout", + "newsletter", + "page", + "press", + "privacy", + "register", + "search", + "sitemap", + "subscribe", + "terms", +]; + +/// File extensions that are assets rather than pages. +const NON_PAGE_EXTENSIONS: &[&str] = &[ + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".svg", ".ico", ".css", ".js", ".json", + ".xml", ".pdf", ".zip", ".mp4", ".mp3", ".rss", +]; + +/// Bounds on how much of a site a single run will load. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct CrawlBudget { + /// Maximum number of sections to sample. + pub(super) max_sections: usize, + /// Maximum number of pages to load in total, including the root. + pub(super) max_pages: usize, +} + +impl Default for CrawlBudget { + fn default() -> Self { + Self { + max_sections: 8, + max_pages: 17, + } + } +} + +/// One section selected for sampling. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PlannedSection { + /// The first path segment identifying the section (`news`). + pub(super) segment: String, + /// The section landing page, when one was observed. + pub(super) landing: Option, + /// A representative content page inside the section, when one was observed. + pub(super) article: Option, +} + +impl PlannedSection { + /// The pages to load for this section, landing first. + fn targets(&self) -> impl Iterator { + self.landing.iter().chain(self.article.iter()) + } +} + +/// The bounded outcome of planning. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct CrawlPlan { + /// Sections selected for sampling, highest confidence first. + pub(super) sections: Vec, + /// Sections found but dropped because the budget was already spent. + pub(super) dropped_sections: Vec, + /// Human-readable notes about how the plan was reached. + pub(super) notes: Vec, +} + +impl CrawlPlan { + /// Page URLs to load, in crawl order. The root is *not* included — the + /// caller has already collected it in order to plan at all. + pub(super) fn targets(&self) -> Vec { + self.sections + .iter() + .flat_map(PlannedSection::targets) + .cloned() + .collect() + } +} + +/// Evidence gathered about one candidate section before ranking. +#[derive(Debug, Default)] +struct SectionCandidate { + landing: Option, + article: Option, + in_nav: bool, + in_sitemap: bool, + link_count: usize, +} + +impl SectionCandidate { + /// Confidence ordering: corroborated by both sources beats either alone, + /// and navigation beats a sitemap-only hit because navigation is the + /// publisher's own statement of what its sections are. + fn rank(&self) -> u8 { + match (self.in_nav, self.in_sitemap) { + (true, true) => 3, + (true, false) => 2, + (false, true) => 1, + (false, false) => 0, + } + } +} + +/// Plans the crawl from the root page's links and any sitemap entries. +/// +/// `root` bounds the crawl: every candidate must share its origin, which also +/// stops a hostile or misconfigured `robots.txt` from redirecting the crawl (and +/// the operator's cookies) at an unrelated host. +pub(super) fn plan_crawl( + root: &Url, + links: &[CollectedLink], + sitemap_locs: &[String], + budget: CrawlBudget, +) -> CrawlPlan { + let mut candidates: BTreeMap = BTreeMap::new(); + let mut notes = Vec::new(); + + for link in links { + let Some(url) = same_origin_page_url(root, &link.url) else { + continue; + }; + let Some(segment) = first_segment(&url) else { + continue; + }; + let entry = candidates.entry(segment).or_default(); + entry.in_nav |= link.in_nav; + entry.link_count += 1; + record_url(entry, &url); + } + + let mut sitemap_pages = 0_usize; + for loc in sitemap_locs { + let Some(url) = same_origin_page_url(root, loc) else { + continue; + }; + let Some(segment) = first_segment(&url) else { + continue; + }; + sitemap_pages += 1; + let entry = candidates.entry(segment).or_default(); + entry.in_sitemap = true; + record_url(entry, &url); + } + + if !sitemap_locs.is_empty() { + notes.push(format!( + "sitemap contributed {sitemap_pages} same-origin page(s) across {} section(s)", + candidates.values().filter(|c| c.in_sitemap).count() + )); + } + if links.iter().all(|link| !link.in_nav) && !links.is_empty() { + notes.push( + "no navigation links were found; sections were inferred from body links only" + .to_string(), + ); + } + + // Rank before truncating: confidence first, then how heavily the section is + // linked, then the segment name so runs are reproducible. + let mut ranked: Vec<(String, SectionCandidate)> = candidates.into_iter().collect(); + ranked.sort_by(|(left_segment, left), (right_segment, right)| { + right + .rank() + .cmp(&left.rank()) + .then(right.link_count.cmp(&left.link_count)) + .then(left_segment.cmp(right_segment)) + }); + + let mut sections = Vec::new(); + let mut dropped_sections = Vec::new(); + // The root page is already collected and counts against the page budget. + let mut pages_used = 1_usize; + for (segment, candidate) in ranked { + let planned = PlannedSection { + segment: segment.clone(), + landing: candidate.landing, + article: candidate.article, + }; + let cost = planned.targets().count(); + if cost == 0 { + continue; + } + if sections.len() >= budget.max_sections || pages_used + cost > budget.max_pages { + dropped_sections.push(segment); + continue; + } + pages_used += cost; + sections.push(planned); + } + + if !dropped_sections.is_empty() { + notes.push(format!( + "budget reached: {} section(s) not sampled ({}); raise --max-sections/--max-pages to include them", + dropped_sections.len(), + dropped_sections.join(", ") + )); + } + + CrawlPlan { + sections, + dropped_sections, + notes, + } +} + +/// Files a URL as the section's landing page or its representative article. +/// +/// The first candidate of each kind wins, so a run is stable given stable input. +fn record_url(entry: &mut SectionCandidate, url: &Url) { + if segment_count(url) == 1 { + if entry.landing.is_none() { + entry.landing = Some(url.clone()); + } + } else if entry.article.is_none() { + entry.article = Some(url.clone()); + } +} + +/// Parses `raw` against `root` and keeps it only if it is a same-origin page. +/// +/// Rejects other origins, non-HTTP schemes, asset extensions, and paginated or +/// utility paths. Query and fragment are dropped so `/news?page=2` and +/// `/news#top` collapse onto `/news`. +fn same_origin_page_url(root: &Url, raw: &str) -> Option { + let mut url = root.join(raw).ok()?; + if !matches!(url.scheme(), "http" | "https") || url.origin() != root.origin() { + return None; + } + url.set_query(None); + url.set_fragment(None); + + let path = url.path().to_ascii_lowercase(); + if NON_PAGE_EXTENSIONS + .iter() + .any(|extension| path.ends_with(extension)) + { + return None; + } + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.is_empty() { + return None; + } + if NOISE_SEGMENTS.contains(&segments[0]) { + return None; + } + // `/news/page/2` is the same inventory as `/news`, so it is not a second + // sample worth spending a page load on. + if segments.contains(&"page") { + return None; + } + Some(url) +} + +/// The first non-empty path segment, lowercased. +fn first_segment(url: &Url) -> Option { + url.path() + .split('/') + .find(|part| !part.is_empty()) + .map(str::to_ascii_lowercase) +} + +/// Count of non-empty path segments. +fn segment_count(url: &Url) -> usize { + url.path() + .split('/') + .filter(|part| !part.is_empty()) + .count() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root() -> Url { + Url::parse("https://publisher.example/").expect("valid root") + } + + fn nav(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + } + } + + fn body(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: false, + } + } + + fn segments(plan: &CrawlPlan) -> Vec<&str> { + plan.sections + .iter() + .map(|section| section.segment.as_str()) + .collect() + } + + #[test] + fn pairs_a_landing_page_with_an_article_from_the_sitemap() { + let plan = plan_crawl( + &root(), + &[nav("/news")], + &["https://publisher.example/news/story-abc".to_string()], + CrawlBudget::default(), + ); + + assert_eq!(segments(&plan), ["news"]); + let section = &plan.sections[0]; + assert_eq!( + section.landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news") + ); + assert_eq!( + section.article.as_ref().map(Url::as_str), + Some("https://publisher.example/news/story-abc") + ); + assert_eq!(plan.targets().len(), 2, "should load landing then article"); + } + + #[test] + fn cross_origin_candidates_are_dropped() { + // Guards both the sitemap (a `Sitemap:` directive can point anywhere) + // and links: the crawl carries operator cookies, so it must not leave + // the requested origin. + let plan = plan_crawl( + &root(), + &[CollectedLink { + url: "https://tracker.example/news".to_string(), + in_nav: true, + }], + &["https://other.example/deals/x".to_string()], + CrawlBudget::default(), + ); + + assert!( + plan.sections.is_empty(), + "no off-origin section should survive, got {:?}", + segments(&plan) + ); + } + + #[test] + fn utility_paths_and_assets_are_filtered() { + let plan = plan_crawl( + &root(), + &[ + nav("/about-us"), + nav("/search"), + nav("/editorial-policy"), + nav("/logo.png"), + nav("/feed.xml"), + nav("/news/page/2"), + nav("/news"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the real content section should remain" + ); + } + + #[test] + fn query_and_fragment_collapse_onto_one_landing_page() { + let plan = plan_crawl( + &root(), + &[nav("/news?utm_source=x"), nav("/news#top"), nav("/news")], + &[], + CrawlBudget::default(), + ); + + assert_eq!(segments(&plan), ["news"]); + assert_eq!( + plan.sections[0].landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news"), + "tracking query and fragment should be stripped" + ); + } + + #[test] + fn nav_and_sitemap_corroboration_outranks_either_alone() { + let plan = plan_crawl( + &root(), + &[nav("/features"), body("/reviews")], + &[ + "https://publisher.example/features/story".to_string(), + "https://publisher.example/deals/x".to_string(), + ], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan)[0], + "features", + "nav + sitemap should rank first, got {:?}", + segments(&plan) + ); + } + + #[test] + fn budget_truncates_and_reports_what_was_dropped() { + let links: Vec = ["a", "b", "c", "d"] + .iter() + .map(|segment| nav(&format!("/{segment}"))) + .collect(); + + let plan = plan_crawl( + &root(), + &links, + &[], + CrawlBudget { + max_sections: 2, + max_pages: 17, + }, + ); + + assert_eq!(plan.sections.len(), 2, "section cap should be honoured"); + assert_eq!(plan.dropped_sections.len(), 2); + assert!( + plan.notes + .iter() + .any(|note| note.contains("budget reached")), + "dropping sections must be reported, not silent: {:?}", + plan.notes + ); + } + + #[test] + fn page_budget_counts_the_already_collected_root() { + // max_pages = 3 leaves room for exactly one landing+article pair on top + // of the root page the caller already loaded. + let plan = plan_crawl( + &root(), + &[nav("/news"), nav("/deals")], + &[ + "https://publisher.example/news/a".to_string(), + "https://publisher.example/deals/b".to_string(), + ], + CrawlBudget { + max_sections: 8, + max_pages: 3, + }, + ); + + assert_eq!( + plan.targets().len(), + 2, + "root + 2 pages fills max_pages = 3" + ); + assert_eq!(plan.dropped_sections.len(), 1); + } + + #[test] + fn body_only_links_still_yield_sections_with_a_note() { + let plan = plan_crawl( + &root(), + &[body("/news"), body("/deals")], + &[], + CrawlBudget::default(), + ); + + assert_eq!(segments(&plan), ["deals", "news"]); + assert!( + plan.notes + .iter() + .any(|note| note.contains("no navigation links")), + "a nav-less page should say so: {:?}", + plan.notes + ); + } + + #[test] + fn empty_input_plans_nothing_rather_than_panicking() { + let plan = plan_crawl(&root(), &[], &[], CrawlBudget::default()); + + assert!(plan.sections.is_empty()); + assert!(plan.targets().is_empty()); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index d3125a1f5..8d2491801 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -1,6 +1,7 @@ mod analyzer; pub(crate) mod browser_collector; pub(crate) mod collector; +mod crawl_plan; mod gpt_slots; mod slot_toml; mod validate; @@ -667,6 +668,8 @@ mod tests { resource_type: Some("script".to_string()), }], gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), } } @@ -953,6 +956,8 @@ mod tests { resource_type: Some("fetch".to_string()), }], gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; From 73ac39c1cecfd9b38edccc4e479aefd41312b8d8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:04:55 +0530 Subject: [PATCH 257/395] Accumulate cross-page slot evidence for ad-template generate Template inference needs the set of observations per slot, not one snapshot: a single page cannot distinguish a literal ad-unit path from a templated one, so the divergence across pages is the only signal available. Add the table that holds it. Nothing calls this yet. Slots are keyed on the normalized div stem, since raw GPT div ids carry per-render framework hashes and would otherwise look like a new slot on every page. Three reconciliations happen here and nowhere else: - Formats union across pages. A size that renders only on article pages, such as a 300x600 rail, has to survive alongside the homepage's sizes; taking the first page's list would silently narrow the slot. - Divergent unit paths are retained as separate rows rather than collapsed, because discarding them is what makes templating impossible. - Network ids must agree. Two GAM networks in one crawl means the pages are not one property, so this is a hard error naming both rather than a guess that would bid against the wrong inventory. Pages that yield no slots are recorded rather than dropped, so a caller can recognise a bot challenge serving interstitials and refuse to write a half-empty config. --- .../src/commands/audit/generate/evidence.rs | 372 ++++++++++++++++++ .../src/commands/audit/generate/mod.rs | 1 + 2 files changed, 373 insertions(+) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/evidence.rs diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs new file mode 100644 index 000000000..e1b1bf81b --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -0,0 +1,372 @@ +//! Cross-page slot evidence: what each slot looked like on every page it was +//! observed on. +//! +//! A single page cannot distinguish a literal ad-unit path from a templated one, +//! so inference needs the *set* of observations per slot rather than one +//! snapshot. This module accumulates that set and is deliberately the only place +//! that reconciles a slot seen more than once: +//! +//! - **Formats union.** A size that appears only on article pages (a 300x600 +//! rail, say) must survive alongside the homepage's sizes. Taking the first +//! page's formats would silently narrow the slot. +//! - **Unit paths are kept, not collapsed.** Divergence across pages is the +//! signal inference reads; discarding it is what makes templating impossible. +//! - **Network ids must agree.** Two different GAM networks in one crawl means +//! the pages are not one property, and writing either one would be a guess. +//! +//! Slots are keyed on the *normalized div stem* produced by +//! [`discover_gpt_slots`](super::gpt_slots::discover_gpt_slots), because raw GPT +//! div ids carry per-render framework hashes and would otherwise look like a new +//! slot on every page. + +#![allow( + dead_code, + reason = "table is exercised by tests until run_update_slots orchestrates the crawl" +)] + +use std::collections::{BTreeMap, BTreeSet}; + +use super::gpt_slots::DiscoveredSlots; +use crate::error::{CliResult, cli_error}; + +/// One observation of a slot on one page. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct EvidenceRow { + /// The page path the slot was observed on, normalized (leading `/`, no + /// query or fragment). + pub(super) path: String, + /// The literal GAM ad-unit path the live page used for this slot. + pub(super) unit_path: String, +} + +/// Everything observed about one slot across the crawl. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SlotEvidence { + /// Config slot id derived from the div stem. + pub(super) id: String, + /// Normalized div stem, used as the runtime `div_id` prefix. + pub(super) div_id: String, + /// Union of every pixel size observed for this slot, smallest first. + pub(super) formats: BTreeSet<(u32, u32)>, + /// Whether any page carrying this slot showed header-bidding signals. + pub(super) has_prebid: bool, + /// Distinct `(path, unit_path)` observations, in a stable order. + pub(super) rows: BTreeSet, +} + +impl SlotEvidence { + /// The distinct literal unit paths observed for this slot. + pub(super) fn unit_paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.unit_path.as_str()).collect() + } + + /// The distinct page paths this slot was observed on. + pub(super) fn paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.path.as_str()).collect() + } +} + +/// Slot evidence accumulated across every collected page. +#[derive(Debug, Clone, Default)] +pub(super) struct EvidenceTable { + slots: BTreeMap, + /// Div stems in first-seen order, so generated config keeps crawl order + /// rather than alphabetical order. + order: Vec, + network_ids: BTreeSet, + /// Every page path folded in, including those that yielded no slots. + pages: BTreeSet, + /// Page paths that produced no slot evidence at all. + empty_pages: BTreeSet, +} + +impl EvidenceTable { + /// Folds one page's discovered slots into the table. + /// + /// `path` is the page's normalized request path; it is what page patterns + /// and `{section}` derivation are computed from later, so it must be the + /// post-redirect path actually audited. + pub(super) fn fold_page(&mut self, path: &str, discovered: &DiscoveredSlots) { + self.pages.insert(path.to_string()); + if let Some(network_id) = &discovered.gam_network_id { + self.network_ids.insert(network_id.clone()); + } + if discovered.slots.is_empty() { + self.empty_pages.insert(path.to_string()); + return; + } + + for slot in &discovered.slots { + let entry = self.slots.entry(slot.div_id.clone()).or_insert_with(|| { + self.order.push(slot.div_id.clone()); + SlotEvidence { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + formats: BTreeSet::new(), + has_prebid: false, + rows: BTreeSet::new(), + } + }); + // Union rather than replace: a size seen only on one page type is + // still a size this slot serves. + entry.formats.extend(slot.formats.iter().copied()); + entry.has_prebid |= slot.has_prebid; + entry.rows.insert(EvidenceRow { + path: path.to_string(), + unit_path: slot.gam_unit_path.clone(), + }); + } + } + + /// Slots in first-seen order. + pub(super) fn slots(&self) -> impl Iterator { + self.order + .iter() + .filter_map(|div_id| self.slots.get(div_id)) + } + + /// Number of distinct slots observed. + pub(super) fn slot_count(&self) -> usize { + self.slots.len() + } + + /// Every page path folded in, whether or not it yielded slots. + pub(super) fn pages(&self) -> &BTreeSet { + &self.pages + } + + /// Page paths that produced no slot evidence. + /// + /// A high proportion of these is the signature of a bot challenge serving + /// interstitials instead of the real site, which is worth refusing to write + /// from rather than persisting a half-empty config. + pub(super) fn empty_pages(&self) -> &BTreeSet { + &self.empty_pages + } + + /// Whether any slot was observed at all. + pub(super) fn is_empty(&self) -> bool { + self.slots.is_empty() + } + + /// The single GAM network id observed across the crawl. + /// + /// # Errors + /// + /// Returns an error when pages disagreed. Two networks in one crawl means + /// the pages are not one property (a syndicated subdomain, a child network, + /// an off-origin redirect that slipped through), and picking either would be + /// a guess that silently bids against the wrong inventory. + pub(super) fn network_id(&self) -> CliResult> { + let mut found = self.network_ids.iter(); + let Some(first) = found.next() else { + return Ok(None); + }; + if self.network_ids.len() > 1 { + let all: Vec<&str> = self.network_ids.iter().map(String::as_str).collect(); + return cli_error(format!( + "the crawled pages reported more than one GAM network id ({}); \ + they do not appear to be one property, so no network id can be \ + chosen safely. Audit a single property, or pass explicit URLs", + all.join(", ") + )); + } + Ok(Some(first.clone())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// One live slot as `(unit path, div id, sizes)`. + type SlotFixture<'a> = (&'a str, &'a str, &'a [(u32, u32)]); + + fn page(slots: &[SlotFixture<'_>], has_prebid: bool) -> DiscoveredSlots { + let registry: Vec = slots + .iter() + .map(|(unit_path, div_id, sizes)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: sizes.to_vec(), + }) + .collect(); + discover_gpt_slots(®istry, &[], has_prebid) + } + + #[test] + fn formats_union_across_pages_instead_of_first_seen_winning() { + // The 300x600 rail only ever renders on article pages. Keeping the + // homepage's format list alone would silently narrow the slot. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-rail", &[(300, 250)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-rail", &[(300, 600)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.formats.iter().copied().collect::>(), + [(300, 250), (300, 600)], + "both pages' sizes should survive" + ); + assert_eq!(table.slot_count(), 1, "one div stem is one slot"); + } + + #[test] + fn divergent_unit_paths_are_preserved_as_separate_rows() { + // This divergence is the entire signal template inference reads. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.unit_paths().into_iter().collect::>(), + ["/123/site/home", "/123/site/news"], + "both observed unit paths must be retained" + ); + assert_eq!( + slot.paths().into_iter().collect::>(), + ["/", "/news/story"] + ); + } + + #[test] + fn repeated_identical_observations_collapse() { + let mut table = EvidenceTable::default(); + let observed = page(&[("/123/site/home", "ad-header", &[(728, 90)])], false); + table.fold_page("/", &observed); + table.fold_page("/", &observed); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!(slot.rows.len(), 1, "the same page twice is one observation"); + } + + #[test] + fn prebid_is_sticky_once_any_page_shows_it() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], true), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert!( + slot.has_prebid, + "a slot proven to run prebid on any page runs prebid" + ); + } + + #[test] + fn slots_keep_first_seen_order_not_alphabetical_order() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page( + &[ + ("/123/site/home", "zeta-slot", &[(728, 90)]), + ("/123/site/home", "alpha-slot", &[(300, 250)]), + ], + false, + ), + ); + + let ids: Vec<&str> = table.slots().map(|slot| slot.div_id.as_str()).collect(); + assert_eq!( + ids, + ["zeta-slot", "alpha-slot"], + "generated config should follow crawl order" + ); + } + + #[test] + fn conflicting_network_ids_are_a_hard_error() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/111/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/222/site/news", "ad-header", &[(728, 90)])], false), + ); + + let error = table + .network_id() + .expect_err("two networks in one crawl should not resolve"); + + let rendered = format!("{error:?}"); + assert!( + rendered.contains("111") && rendered.contains("222"), + "the error should name both observed ids, got {rendered}" + ); + } + + #[test] + fn agreeing_network_ids_resolve_to_one_value() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + assert_eq!( + table.network_id().expect("agreeing ids should resolve"), + Some("123".to_string()) + ); + } + + #[test] + fn pages_without_slots_are_recorded_for_challenge_detection() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page("/blocked", &page(&[], false)); + + assert_eq!( + table + .empty_pages() + .iter() + .map(String::as_str) + .collect::>(), + ["/blocked"], + "a slot-less page must be visible to the caller, not silently dropped" + ); + assert_eq!( + table.pages().len(), + 2, + "every folded page should be counted" + ); + } + + #[test] + fn empty_table_resolves_no_network_id_rather_than_erroring() { + let table = EvidenceTable::default(); + + assert!(table.is_empty()); + assert_eq!(table.network_id().expect("empty is not a conflict"), None); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 8d2491801..649a57047 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -2,6 +2,7 @@ mod analyzer; pub(crate) mod browser_collector; pub(crate) mod collector; mod crawl_plan; +mod evidence; mod gpt_slots; mod slot_toml; mod validate; From f32a1acf5d175d2fcc005eb4ac6a4e57043d3ccd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:12:26 +0530 Subject: [PATCH 258/395] Infer section ad-unit templates from cross-page evidence Adds the inference that turns literal scraped ad-unit paths into a `{network_id}`/`{section}` template plus the section policy it depends on. Nothing calls this yet. A wrong template makes a publisher bid against inventory that does not exist, which is worse than a narrow literal path, so this refuses rather than guesses. Three rules carry that: - `{network_id}` binds positionally to unit segment 0 and only when that segment already equals the resolved id. Substring replacement would rewrite `/123/sports123/home` into `/{network_id}/sports{network_id}/home`. - Exactly one unit segment may vary. Zero proves nothing and stays literal; two means the unit tracks a dimension the request path cannot supply, such as a device or geo split, and is refused with that reason. - Two pages must witness both a different derived section and a different unit segment before anything is templated. Round-trip verification cannot supply this: a single observation is reproduced equally well by a literal path, a `{network_id}`-only template, and a `{section}` template, so only variation distinguishes them. `section_segment` is chosen by partitioning observations into pages that have a section segment and pages that do not, the latter fixing `section_root`. An index that cannot be witnessed is rejected, an unwitnessed root leaves the path literal rather than guessing, and two indices that both fit are ambiguous and template nothing. Every accepted template is then replayed through the runtime's own `render_gam_unit_path` and `derive_section` against every observation, so a section slug the path cannot reproduce is caught and downgraded. `derive_section` becomes public for exactly this: the check has to use the runtime's derivation rather than a second implementation that could drift from it. --- .../src/commands/audit/generate/mod.rs | 1 + .../commands/audit/generate/unit_template.rs | 841 ++++++++++++++++++ .../src/creative_opportunities.rs | 6 +- 3 files changed, 847 insertions(+), 1 deletion(-) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 649a57047..c90235649 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -5,6 +5,7 @@ mod crawl_plan; mod evidence; mod gpt_slots; mod slot_toml; +mod unit_template; mod validate; use std::collections::BTreeSet; diff --git a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs new file mode 100644 index 000000000..790a89859 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs @@ -0,0 +1,841 @@ +//! Infers a `{network_id}`/`{section}` ad-unit template from observed evidence. +//! +//! The generator otherwise writes the literal path each page happened to +//! request, which pins a slot to the one section it was scraped from. A template +//! generalizes across sections — but a *wrong* template makes the publisher bid +//! against inventory that does not exist, which is worse than a narrow literal. +//! So this module is built to refuse rather than guess. +//! +//! Three rules do the load-bearing work: +//! +//! 1. **Positional binding.** `{network_id}` is bound to unit segment 0 and only +//! if that segment is the resolved network id. Substring replacement would +//! corrupt `/123/sports123/home` into `/{network_id}/sports{network_id}/home`. +//! 2. **Exactly one varying segment.** Zero means nothing was proven and the +//! path stays literal; two means the unit varies along a dimension the +//! request path cannot supply (device, geo, experiment), so it is refused. +//! 3. **The witness rule.** Two pages must show *different* derived sections +//! *and* different unit segments. Without it a single-page crawl is +//! indistinguishable from a static path — literal, `{network_id}`-only and +//! `{section}` all reproduce one observation equally well, and round-trip +//! verification cannot tell them apart. Only variation can. +//! +//! Every accepted template is then replayed through the runtime's own +//! [`render_gam_unit_path`](CreativeOpportunitySlot::render_gam_unit_path) and +//! [`derive_section`] against every observation. A template that does not +//! reproduce what the live page actually requested is downgraded, not written. + +#![allow( + dead_code, + reason = "inference is exercised by tests until run_update_slots orchestrates the crawl" +)] + +use std::collections::{BTreeMap, BTreeSet}; + +use trusted_server_core::creative_opportunities::{CreativeOpportunitySlot, derive_section}; + +use super::evidence::{EvidenceTable, SlotEvidence}; +use super::slot_toml::toml_string; + +/// Candidate `section_segment` values considered, `0..=MAX_SECTION_SEGMENT`. +/// +/// A locale-prefixed site (`/en/news/story`) needs 1. Beyond 2 the "section" is +/// no longer a taxonomy the operator would recognise, and every extra candidate +/// is another chance for two indices to both fit and force a refusal. +const MAX_SECTION_SEGMENT: usize = 2; + +/// The config-level section policy an inferred template depends on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SectionPolicy { + /// Value substituted for `{section}` on paths with no section segment. + pub(super) section_root: String, + /// Index of the path segment `{section}` is taken from. + pub(super) section_segment: usize, +} + +/// What to write for one slot's `gam_unit_path`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum SlotDecision { + /// Write this templated path; it reproduced every observation. + Template(String), + /// Write this literal path; nothing generalizable was proven. + Literal(String), + /// Write no path at all — the observations cannot be represented. + Refuse { + /// Operator-facing explanations, one per reason. + reasons: Vec, + }, +} + +/// The outcome of inference across the whole evidence table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct InferenceOutcome { + /// Section policy to write, present only when some slot templated. + pub(super) policy: Option, + /// Per-slot decision, keyed by div stem, in evidence order. + pub(super) decisions: Vec<(String, SlotDecision)>, + /// Operator-facing notes about why inference went the way it did. + pub(super) diagnostics: Vec, +} + +impl InferenceOutcome { + /// The decision for a slot, by div stem. + pub(super) fn decision(&self, div_id: &str) -> Option<&SlotDecision> { + self.decisions + .iter() + .find(|(key, _)| key == div_id) + .map(|(_, decision)| decision) + } +} + +/// Per-slot analysis under one candidate `section_segment`. +#[derive(Debug, Clone, PartialEq, Eq)] +enum SlotAnalysis { + /// Templatable: unit segment `varying` tracks the derived section, and root + /// pages agreed on `section_root`. + Templatable { + varying: usize, + section_root: String, + }, + /// The unit path never varied, so nothing about `{section}` was proven. + Static, + /// Cannot be represented; carries the operator-facing reason. + Refuse(String), + /// Would be templatable but no root page was observed, so `section_root` + /// is undetermined under this candidate. + RootUnwitnessed, +} + +/// Infers unit-path templates for every slot in `table`. +/// +/// `network_id` is the resolved GAM network id; `{network_id}` is only ever +/// bound to a unit segment that already equals it. +pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> InferenceOutcome { + let slots: Vec<&SlotEvidence> = table.slots().collect(); + let mut diagnostics = Vec::new(); + + // Evaluate every candidate index independently; ambiguity between two that + // both fit is a refusal, not a preference for the smaller one. + let mut qualifying: Vec<(usize, String, BTreeMap)> = Vec::new(); + for segment in 0..=MAX_SECTION_SEGMENT { + let analyses: BTreeMap = slots + .iter() + .map(|slot| (slot.div_id.clone(), analyse_slot(slot, network_id, segment))) + .collect(); + + let roots: BTreeSet<&str> = analyses + .values() + .filter_map(|analysis| match analysis { + SlotAnalysis::Templatable { section_root, .. } => Some(section_root.as_str()), + _ => None, + }) + .collect(); + // Slots must agree: `section_root` is one config-level value, so two + // slots claiming different roots means this index is not the real one. + let Some(root) = roots.iter().next().copied() else { + continue; + }; + if roots.len() > 1 { + continue; + } + if !witnessed(&slots, &analyses, segment) { + continue; + } + qualifying.push((segment, root.to_string(), analyses)); + } + + let chosen = match qualifying.len() { + 0 => None, + 1 => qualifying.into_iter().next(), + _ => { + let indices: Vec = qualifying + .iter() + .map(|(segment, _, _)| segment.to_string()) + .collect(); + diagnostics.push(format!( + "more than one section_segment ({}) explains the observed ad-unit paths \ + equally well, so no template can be chosen safely; keeping literal paths", + indices.join(", ") + )); + None + } + }; + + let Some((section_segment, section_root, analyses)) = chosen else { + if diagnostics.is_empty() { + diagnostics.push( + "no ad-unit path varied by page section across the crawl, so paths were kept \ + literal; crawl more sections to enable a {section} template" + .to_string(), + ); + } + return InferenceOutcome { + policy: None, + decisions: literal_decisions(&slots), + diagnostics, + }; + }; + + let mut decisions = Vec::with_capacity(slots.len()); + let mut templated = 0_usize; + for slot in &slots { + let analysis = analyses + .get(&slot.div_id) + .cloned() + .unwrap_or(SlotAnalysis::Static); + let decision = match analysis { + SlotAnalysis::Templatable { varying, .. } => { + let template = build_template(slot, varying); + match verify_round_trip(&template, slot, network_id, §ion_root, section_segment) + { + Ok(()) => { + templated += 1; + SlotDecision::Template(template) + } + Err(reason) => { + diagnostics.push(format!( + "slot `{}` template `{template}` did not reproduce the observed \ + ad-unit paths ({reason}); keeping the literal path", + slot.id + )); + literal_decision(slot) + } + } + } + SlotAnalysis::Static | SlotAnalysis::RootUnwitnessed => literal_decision(slot), + SlotAnalysis::Refuse(reason) => SlotDecision::Refuse { + reasons: vec![reason], + }, + }; + decisions.push((slot.div_id.clone(), decision)); + } + + if templated == 0 { + return InferenceOutcome { + policy: None, + decisions, + diagnostics, + }; + } + + diagnostics.push(format!( + "inferred section_segment = {section_segment} and section_root = \"{section_root}\" \ + from {} page(s); {templated} slot(s) templated", + table.pages().len() + )); + InferenceOutcome { + policy: Some(SectionPolicy { + section_root, + section_segment, + }), + decisions, + diagnostics, + } +} + +/// Whether the accepted analyses actually witnessed section variation. +/// +/// Requires two rows with both a different derived section and a different +/// value in the varying unit segment. Round-trip verification cannot supply +/// this: one observation is reproduced equally well by a literal path, a +/// `{network_id}`-only template, and a `{section}` template. +fn witnessed( + slots: &[&SlotEvidence], + analyses: &BTreeMap, + section_segment: usize, +) -> bool { + for slot in slots { + let Some(SlotAnalysis::Templatable { + varying, + section_root, + }) = analyses.get(&slot.div_id) + else { + continue; + }; + let mut sections = BTreeSet::new(); + let mut units = BTreeSet::new(); + for row in &slot.rows { + sections.insert(derive_section(&row.path, section_root, section_segment)); + if let Some(value) = segment_at(&row.unit_path, *varying) { + units.insert(value.to_string()); + } + } + if sections.len() >= 2 && units.len() >= 2 { + return true; + } + } + false +} + +/// Checks the properties of a slot's observations that do not depend on which +/// `section_segment` is being considered. +/// +/// Kept separate because these refusals are final: no candidate index can +/// rescue a slot whose observations are not one template with a single hole in +/// them, and the operator needs the specific reason rather than a generic one. +/// +/// Returns the single varying unit segment, `None` when nothing varied, or the +/// reason the observations cannot be represented at all. +fn structural_check(slot: &SlotEvidence) -> Result, String> { + // One page reporting two different ad-unit paths for the same slot means the + // unit varies along something the request path cannot express — a device or + // geo split, or two profiles disagreeing. Nothing here can represent that. + let mut per_path: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for row in &slot.rows { + per_path + .entry(row.path.as_str()) + .or_default() + .insert(row.unit_path.as_str()); + } + if let Some((path, units)) = per_path.iter().find(|(_, units)| units.len() > 1) { + let observed: Vec<&str> = units.iter().copied().collect(); + return Err(format!( + "page `{path}` requested more than one ad-unit path for this slot ({}); \ + the unit varies by something the request path cannot derive", + observed.join(", ") + )); + } + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + let Some(first) = split.first() else { + return Ok(None); + }; + // Differing shapes are not one template with a hole in it. + if split.iter().any(|parts| parts.len() != first.len()) { + return Err( + "the observed ad-unit paths have different segment counts, so they are not \ + one template" + .to_string(), + ); + } + + let varying: Vec = (0..first.len()) + .filter(|index| { + split + .iter() + .map(|parts| parts[*index]) + .collect::>() + .len() + > 1 + }) + .collect(); + match varying.len() { + 0 => Ok(None), + 1 if varying[0] == 0 => { + Err("the network-id segment of the ad-unit path varied across pages".to_string()) + } + 1 => Ok(Some(varying[0])), + count => Err(format!( + "{count} ad-unit segments vary across pages, so the path does not track the \ + page section alone" + )), + } +} + +/// Analyses one slot under a candidate `section_segment`. +fn analyse_slot(slot: &SlotEvidence, network_id: &str, section_segment: usize) -> SlotAnalysis { + let varying = match structural_check(slot) { + Err(reason) => return SlotAnalysis::Refuse(reason), + Ok(None) => return SlotAnalysis::Static, + Ok(Some(varying)) => varying, + }; + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + // `{network_id}` binds positionally and only to the resolved id. Substring + // replacement would rewrite an unrelated segment that merely contains it. + if split.first().and_then(|parts| parts.first()) != Some(&network_id) { + return SlotAnalysis::Static; + } + + // Partition observations into pages that have a section segment and pages + // that do not; the latter are what determine `section_root`. + let mut root_values = BTreeSet::new(); + for (row, parts) in slot.rows.iter().zip(split.iter()) { + let observed = parts[varying]; + if path_segments(&row.path).len() > section_segment { + // The empty root is unused here: the path has this segment. + if derive_section(&row.path, "", section_segment) != observed { + return SlotAnalysis::Static; + } + } else { + root_values.insert(observed); + } + } + + let mut roots = root_values.into_iter(); + let Some(section_root) = roots.next() else { + // Without a root observation, `section_root` would be a guess that + // silently mis-renders every short path. + return SlotAnalysis::RootUnwitnessed; + }; + if roots.next().is_some() { + return SlotAnalysis::Static; + } + // A root that is not `[A-Za-z0-9_-]+` makes any `{section}` template fail + // config load; catch it here rather than at push time. + if section_root.is_empty() + || !section_root + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-') + { + return SlotAnalysis::Static; + } + + SlotAnalysis::Templatable { + varying, + section_root: section_root.to_string(), + } +} + +/// Builds the template text by substituting the two proven placeholders. +fn build_template(slot: &SlotEvidence, varying: usize) -> String { + let first = slot + .rows + .iter() + .next() + .map(|row| row.unit_path.as_str()) + .unwrap_or_default(); + let rendered: Vec = segments(first) + .into_iter() + .enumerate() + .map(|(index, value)| { + if index == 0 { + "{network_id}".to_string() + } else if index == varying { + "{section}".to_string() + } else { + value.to_string() + } + }) + .collect(); + format!("/{}", rendered.join("/")) +} + +/// Replays `template` through the runtime renderer against every observation. +/// +/// This is the gate that catches a section slug the path cannot reproduce — a +/// publisher whose `/car-research` pages request `.../carresearch`, say, where +/// the derived section and the observed segment differ. +fn verify_round_trip( + template: &str, + slot: &SlotEvidence, + network_id: &str, + section_root: &str, + section_segment: usize, +) -> Result<(), String> { + let probe = probe_slot(template)?; + for row in &slot.rows { + let section = derive_section(&row.path, section_root, section_segment); + match probe.render_gam_unit_path(network_id, §ion) { + Some(rendered) if rendered == row.unit_path => {} + Some(rendered) => { + return Err(format!( + "on `{}` it renders `{rendered}` but the page requested `{}`", + row.path, row.unit_path + )); + } + None => { + return Err(format!( + "on `{}` it renders past the GAM ad-unit path byte limit", + row.path + )); + } + } + } + Ok(()) +} + +/// Builds a throwaway slot carrying `template`, for rendering only. +/// +/// Deserializing is how the runtime itself builds slots, so this exercises the +/// same template parsing rather than a parallel implementation. +fn probe_slot(template: &str) -> Result { + let document = format!( + "id = \"probe\"\ngam_unit_path = {}\npage_patterns = [\"/\"]\n\ + formats = [{{ width = 1, height = 1 }}]\n", + toml_string(template) + ); + toml::from_str::(&document) + .map_err(|error| format!("template is not representable in config: {error}")) +} + +/// The decision for a slot no template was proven for. +/// +/// A structural refusal wins over the generic "several paths" message, so the +/// operator sees *why* the slot could not be represented (a device split, an +/// extra varying dimension) rather than only that it could not. +fn literal_decision(slot: &SlotEvidence) -> SlotDecision { + if let Err(reason) = structural_check(slot) { + return SlotDecision::Refuse { + reasons: vec![reason], + }; + } + let units = slot.unit_paths(); + let mut found = units.iter(); + match (found.next(), found.next()) { + (Some(only), None) => SlotDecision::Literal((*only).to_string()), + (Some(_), Some(_)) => SlotDecision::Refuse { + reasons: vec![format!( + "the slot used several ad-unit paths ({}) and none generalized, so no \ + single literal path is correct", + units.into_iter().collect::>().join(", ") + )], + }, + _ => SlotDecision::Refuse { + reasons: vec!["no ad-unit path was observed for this slot".to_string()], + }, + } +} + +fn literal_decisions(slots: &[&SlotEvidence]) -> Vec<(String, SlotDecision)> { + slots + .iter() + .map(|slot| (slot.div_id.clone(), literal_decision(slot))) + .collect() +} + +/// Non-empty path segments of an ad-unit path. +fn segments(unit_path: &str) -> Vec<&str> { + unit_path + .split('/') + .filter(|part| !part.is_empty()) + .collect() +} + +/// Non-empty path segments of a request path. +fn path_segments(path: &str) -> Vec<&str> { + path.split('/').filter(|part| !part.is_empty()).collect() +} + +/// The ad-unit path segment at `index`, if present. +fn segment_at(unit_path: &str, index: usize) -> Option<&str> { + segments(unit_path).into_iter().nth(index) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// Folds `(path, unit_path)` observations for one div into a table. + fn table_for(div_id: &str, observations: &[(&str, &str)]) -> EvidenceTable { + let mut table = EvidenceTable::default(); + for (path, unit_path) in observations { + let registry = vec![CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: div_id.to_string(), + sizes: vec![(728, 90)], + }]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + table + } + + fn only_decision(outcome: &InferenceOutcome) -> &SlotDecision { + assert_eq!(outcome.decisions.len(), 1, "fixture should have one slot"); + &outcome.decisions[0].1 + } + + #[test] + fn templates_a_section_varying_unit_path() { + // The shape the operator writes by hand today. + let table = table_for( + "ad-header", + &[ + ("/", "/88059007/autoblog/homepage"), + ("/news/story-abc", "/88059007/autoblog/news"), + ("/deals/thing", "/88059007/autoblog/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "88059007"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }) + ); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/autoblog/{section}".to_string()) + ); + } + + #[test] + fn a_single_page_never_templates() { + // Literal, {network_id}-only and {section} all reproduce one observation, + // so only variation can distinguish them. This is the witness rule. + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/news".to_string()) + ); + } + + #[test] + fn a_static_unit_path_across_sections_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/fixed"), + ("/news/story", "/123/site/fixed"), + ("/deals/x", "/123/site/fixed"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None, "nothing varied, so nothing is proven"); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/fixed".to_string()) + ); + } + + #[test] + fn a_device_split_is_refused_rather_than_guessed() { + // Two units for the SAME path: the desktop/mobile cross-check surfaces + // here, and the request path cannot express the difference. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/news/story", "/123/mobile/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!( + "a device split must refuse, got {:?}", + only_decision(&outcome) + ); + }; + assert!( + reasons[0].contains("more than one ad-unit path"), + "reason should name the conflict, got {reasons:?}" + ); + } + + #[test] + fn two_varying_segments_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/deals/x", "/123/mobile/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("two varying dimensions must refuse"); + }; + assert!( + reasons[0].contains("segments vary"), + "reason should name the extra dimension, got {reasons:?}" + ); + } + + #[test] + fn a_slug_the_path_cannot_reproduce_stays_literal() { + // `/car-research` requests `.../carresearch`: the derived section and + // the observed segment differ, so the template would render the wrong + // unit. Round-trip verification is what catches this. + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news"), + ("/car-research/x", "/123/site/carresearch"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "a section whose slug is not derivable must not template" + ); + assert!(matches!( + only_decision(&outcome), + SlotDecision::Refuse { .. } + )); + } + + #[test] + fn an_unwitnessed_root_does_not_template() { + // Every crawled page had a section, so `section_root` would be a guess + // that silently mis-renders the homepage. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/site/news"), + ("/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + let SlotDecision::Refuse { .. } = only_decision(&outcome) else { + panic!("two literal paths and no template is not representable as one literal"); + }; + } + + #[test] + fn a_locale_prefixed_site_infers_the_deeper_segment() { + let table = table_for( + "ad-header", + &[ + ("/en", "/123/site/homepage"), + ("/en/news/story", "/123/site/news"), + ("/en/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }), + "the locale prefix should push the section one segment deeper" + ); + } + + #[test] + fn network_id_is_bound_positionally_not_by_substring() { + // `sports123` merely contains the network id; substring replacement + // would corrupt it into `sports{network_id}`. + let table = table_for( + "ad-header", + &[ + ("/", "/123/sports123/homepage"), + ("/news/story", "/123/sports123/news"), + ("/deals/x", "/123/sports123/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/sports123/{section}".to_string()), + "only segment 0 may become {{network_id}}" + ); + } + + #[test] + fn a_unit_path_not_starting_with_the_network_id_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/999/site/homepage"), + ("/news/story", "/999/site/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "segment 0 must equal the resolved network id" + ); + } + + #[test] + fn differing_segment_counts_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news/extra"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("differing shapes are not one template"); + }; + assert!( + reasons[0].contains("segment counts"), + "reason should name the shape mismatch, got {reasons:?}" + ); + } + + #[test] + fn a_static_slot_stays_literal_alongside_a_templated_one() { + let mut table = EvidenceTable::default(); + for (path, section_unit) in [ + ("/", "homepage"), + ("/news/story", "news"), + ("/deals/x", "deals"), + ] { + let registry = vec![ + CollectedGptSlot { + gam_unit_path: format!("/123/site/{section_unit}"), + div_id: "ad-header".to_string(), + sizes: vec![(728, 90)], + }, + CollectedGptSlot { + gam_unit_path: "/123/site/sticky".to_string(), + div_id: "ad-sticky".to_string(), + sizes: vec![(300, 250)], + }, + ]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + + let outcome = infer_unit_templates(&table, "123"); + + assert!(outcome.policy.is_some(), "the varying slot should template"); + assert_eq!( + outcome.decision("ad-header"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )) + ); + assert_eq!( + outcome.decision("ad-sticky"), + Some(&SlotDecision::Literal("/123/site/sticky".to_string())), + "a genuinely static slot must not be dragged into the template" + ); + } + + #[test] + fn diagnostics_explain_why_nothing_templated() { + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("crawl more sections")), + "the operator should learn why, got {:?}", + outcome.diagnostics + ); + } +} diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index f63f92d8e..c90c8bd63 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -171,8 +171,12 @@ fn sanitize_section(segment: &str) -> String { /// The path is used **raw** (not percent-decoded) so this stays consistent with /// how [`page_patterns`](CreativeOpportunitySlot::page_patterns) glob-match the /// same path — e.g. `/new%20s` yields `new_20s`, never the decoded `new_s`. +/// +/// Public so operator tooling that *infers* a `{section}` template from observed +/// ad-unit paths can check its inference against the exact derivation the +/// runtime will perform, rather than reimplementing the sanitization rules. #[must_use] -fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { +pub fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { match path .split('/') .filter(|segment| !segment.is_empty()) From deced196987f151c94d7246b5e0198a5d82f0261 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:19:02 +0530 Subject: [PATCH 259/395] Let the ad-template writer express section policy and per-section patterns Two gaps between what inference produces and what the writer could put on disk. Nothing calls the new code yet. `page_patterns` expands the paths a slot was observed on into globs. Each witnessed section contributes a pair, because one glob cannot cover both halves: `*` crosses `/` in this dialect, so `/news/*` matches `/news/a/b` but not the bare `/news` landing page, and emitting only the star form would silently drop the landing page from the slot. Nothing extrapolates past a witnessed section, so a crawl that never visited `/reviews` never claims it. `replace_key_in_section` can only rewrite a key that is already present, so it could not add `section_root` or `section_segment` to a config that predates them, which is every config a first templated run touches. Add `upsert_key_in_section`, which inserts immediately after the section header so the new key lands in the section's scalar block rather than after a subtable, where TOML would read it as belonging to that subtable instead. `splice_creative_slots` now takes the section keys as a struct rather than a bare network id. It omits `section_root` and `section_segment` entirely unless a slot actually templated: both are `deny_unknown_fields` additions, so writing them into a config that does not need them would make it unloadable by an older binary for no benefit. --- .../src/commands/audit/generate/mod.rs | 10 +- .../commands/audit/generate/page_patterns.rs | 137 +++++++++ .../src/commands/audit/generate/slot_toml.rs | 275 ++++++++++++++++-- 3 files changed, 391 insertions(+), 31 deletions(-) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index c90235649..8c1e793ee 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod collector; mod crawl_plan; mod evidence; mod gpt_slots; +mod page_patterns; mod slot_toml; mod unit_template; mod validate; @@ -546,7 +547,14 @@ pub(crate) fn run_update_slots( replace, ); let rendered_slots = render_slots(&merged); - let updated = splice_creative_slots(&existing, network_id.as_deref(), &rendered_slots)?; + let updated = splice_creative_slots( + &existing, + &slot_toml::CreativeSectionKeys { + network_id: network_id.as_deref(), + ..slot_toml::CreativeSectionKeys::default() + }, + &rendered_slots, + )?; // Everything above is derived from a live, page-controlled ad stack, so the // candidate has to clear the runtime's own load path before it can replace diff --git a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs new file mode 100644 index 000000000..70b18a6ae --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs @@ -0,0 +1,137 @@ +//! Derives `page_patterns` globs from the paths a slot was actually observed on. +//! +//! A slot seen on `/news/story-abc` should serve every article in that section, +//! not just that one URL — but nothing here extrapolates beyond a *witnessed* +//! section. Each observed path contributes the section prefix it belongs to and +//! nothing else, so a crawl that never visited `/reviews` never claims it. +//! +//! Each section yields a pair, because one glob cannot cover both halves: +//! `*` crosses `/` in this glob dialect, so `/news/*` matches `/news/a/b` but +//! **not** the bare `/news` landing page. Emitting only the star form silently +//! drops the landing page from the slot. + +#![allow( + dead_code, + reason = "expansion is exercised by tests until run_update_slots orchestrates the crawl" +)] + +use std::collections::BTreeSet; + +/// The root pattern, matching only the site root. +const ROOT_PATTERN: &str = "/"; + +/// Expands observed page paths into the glob set a slot should carry. +/// +/// `section_segment` is the index the section is taken from, matching the +/// config key of the same name: a path is reduced to its first +/// `section_segment + 1` segments, which is the prefix every page of that +/// section shares. Paths shorter than that are root pages and contribute `/`. +/// +/// Results are deduplicated and ordered with `/` first, then alphabetically, so +/// re-running against unchanged evidence produces an unchanged file. +pub(super) fn patterns_for_paths<'a>( + paths: impl IntoIterator, + section_segment: usize, +) -> Vec { + let mut patterns: BTreeSet = BTreeSet::new(); + let mut has_root = false; + + for path in paths { + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.len() <= section_segment { + has_root = true; + continue; + } + let prefix = format!("/{}", segments[..=section_segment].join("/")); + // The landing page and everything beneath it. + patterns.insert(prefix.clone()); + patterns.insert(format!("{prefix}/*")); + } + + let mut out = Vec::with_capacity(patterns.len() + usize::from(has_root)); + if has_root { + out.push(ROOT_PATTERN.to_string()); + } + out.extend(patterns); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_section_article_yields_both_halves_of_the_pair() { + // `/news/*` alone would not match the bare `/news` landing page, because + // `*` crosses `/` but does not match the empty remainder. + let patterns = patterns_for_paths(["/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"]); + } + + #[test] + fn the_root_path_contributes_the_root_pattern_first() { + let patterns = patterns_for_paths(["/deals/x", "/", "/news/y"], 0); + + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "root first, then sections alphabetically" + ); + } + + #[test] + fn a_landing_page_and_its_article_collapse_to_one_pair() { + let patterns = patterns_for_paths(["/news", "/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"], "no duplicate entries"); + } + + #[test] + fn a_locale_prefixed_site_keeps_the_locale_in_the_prefix() { + // section_segment = 1 means the section is the second segment, so the + // shared prefix every page of that section carries includes the locale. + let patterns = patterns_for_paths(["/en/news/story", "/en/deals/x", "/en"], 1); + + assert_eq!( + patterns, + ["/", "/en/deals", "/en/deals/*", "/en/news", "/en/news/*"] + ); + } + + #[test] + fn unwitnessed_sections_are_never_invented() { + let patterns = patterns_for_paths(["/news/story"], 0); + + assert_eq!( + patterns, + ["/news", "/news/*"], + "only the crawled section may appear" + ); + } + + #[test] + fn output_is_stable_regardless_of_input_order() { + let one = patterns_for_paths(["/news/a", "/deals/b", "/"], 0); + let two = patterns_for_paths(["/", "/deals/b", "/news/a"], 0); + + assert_eq!(one, two, "re-running should not reorder the written file"); + } + + #[test] + fn every_emitted_pattern_compiles_as_a_runtime_glob() { + let patterns = patterns_for_paths(["/", "/news/story", "/car-research/x"], 0); + + for pattern in &patterns { + trusted_server_core::creative_opportunities::compile_page_pattern(pattern) + .unwrap_or_else(|error| { + panic!("emitted pattern `{pattern}` must compile: {error}") + }); + } + } + + #[test] + fn no_paths_yield_no_patterns() { + assert!(patterns_for_paths([], 0).is_empty()); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 880ce1cac..a0fc51da7 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -337,11 +337,50 @@ fn toml_inline_value(value: &serde_json::Value) -> String { /// /// If the config has no `[creative_opportunities]` section, a fresh one is /// appended so `generate` works against a config that omits it. +/// The config-level values a splice writes alongside the slot array. +#[derive(Debug, Clone, Default)] +pub(super) struct CreativeSectionKeys<'a> { + /// GAM network id, when one was resolved. + pub(super) network_id: Option<&'a str>, + /// `section_root`, written only when a slot uses a `{section}` template. + pub(super) section_root: Option<&'a str>, + /// `section_segment`, written only alongside `section_root`. + pub(super) section_segment: Option, +} + +impl CreativeSectionKeys<'_> { + /// The `key = value` lines this policy contributes, in config order. + fn lines(&self) -> Vec<(&'static str, String)> { + let mut out = Vec::new(); + if let Some(network_id) = self.network_id { + out.push(( + "gam_network_id", + format!("gam_network_id = {}", toml_string(network_id)), + )); + } + // Both keys are omitted unless a template needs them. They are + // `deny_unknown_fields` additions, so writing them into a config that + // does not need them would make it unloadable by an older binary for no + // benefit. + if let Some(section_root) = self.section_root { + out.push(( + "section_root", + format!("section_root = {}", toml_string(section_root)), + )); + if let Some(segment) = self.section_segment { + out.push(("section_segment", format!("section_segment = {segment}"))); + } + } + out + } +} + pub(super) fn splice_creative_slots( existing: &str, - network_id: Option<&str>, + keys: &CreativeSectionKeys<'_>, rendered_slots: &str, ) -> CliResult { + let network_id = keys.network_id; let rendered = rendered_slots.trim_matches('\n'); let existing = remove_inline_slot_value(existing)?; @@ -378,28 +417,27 @@ pub(super) fn splice_creative_slots( `gam_network_id` to the config and re-run", ); }; + let _ = network_id; let mut result = existing; if !result.is_empty() && !result.ends_with('\n') { result.push('\n'); } result.push_str("\n[creative_opportunities]\n"); - result.push_str(&format!("gam_network_id = {}\n", toml_string(network_id))); + for (_, line) in keys.lines() { + result.push_str(&line); + result.push('\n'); + } result.push_str(rendered); result.push('\n'); return Ok(result); } - // Section exists — update `gam_network_id` (best-effort) and replace slots. + // Section exists — set the scalar keys, then replace the slot array. + // `upsert` rather than `replace`: `section_root`/`section_segment` are new + // keys that a config predating templating simply does not have. let mut document = existing.clone(); - if let Some(network_id) = network_id - && let Ok(updated) = replace_key_in_section( - &document, - "creative_opportunities", - "gam_network_id", - &format!("gam_network_id = {}", toml_string(network_id)), - ) - { - document = updated; + for (key, line) in keys.lines() { + document = upsert_key_in_section(&document, "creative_opportunities", key, &line)?; } let lines: Vec<&str> = document.lines().collect(); @@ -594,6 +632,51 @@ pub(super) fn replace_key_in_section( Ok(output) } +/// Sets `key` in `section`, replacing an existing assignment or inserting one. +/// +/// [`replace_key_in_section`] can only rewrite a key that is already present, so +/// it cannot add `section_root` or `section_segment` to a config that predates +/// them — which is every config a first templated run touches. This inserts +/// immediately after the section header instead, keeping the new key inside the +/// section's scalar block rather than stranding it after a subtable, where TOML +/// would read it as belonging to that subtable. +/// +/// # Errors +/// +/// Returns an error when `section` is not present in the document. +pub(super) fn upsert_key_in_section( + document: &str, + section: &str, + key: &str, + replacement_line: &str, +) -> CliResult { + if let Ok(replaced) = replace_key_in_section(document, section, key, replacement_line) { + return Ok(replaced); + } + + let section_header = format!("[{section}]"); + let Some(header_index) = document + .lines() + .position(|line| is_table_header(line, §ion_header)) + else { + return cli_error(format!( + "failed to update config because section `{section_header}` was not found" + )); + }; + + let mut lines: Vec = document.lines().map(str::to_string).collect(); + lines.insert(header_index + 1, replacement_line.to_string()); + + let mut output = lines.join("\n"); + if document.ends_with('\n') { + output.push('\n'); + } + if uses_crlf(document) { + output = output.replace("\r\n", "\n").replace('\n', "\r\n"); + } + Ok(output) +} + fn is_key_line(trimmed_line: &str, key: &str) -> bool { trimmed_line .strip_prefix(key) @@ -643,6 +726,14 @@ mod tests { render_slots(&merged) } + /// Section keys carrying only a network id, the common test case. + fn network_keys(network_id: &str) -> CreativeSectionKeys<'_> { + CreativeSectionKeys { + network_id: Some(network_id), + ..CreativeSectionKeys::default() + } + } + fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { toml::from_str::(toml_str).expect("valid creative config") } @@ -656,7 +747,7 @@ mod tests { formats = [{ width = 300, height = 250 }]\n\n\ [auction]\nenabled = true\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should splice"); assert!( @@ -686,7 +777,7 @@ mod tests { // produce a document that no longer parses. let existing = "[\"creative_opportunities\"]\ngam_network_id = \"111\"\n"; - let error = splice_creative_slots(existing, Some("222"), &header_rendered()) + let error = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect_err("should refuse an unrecognised section form"); assert!( @@ -699,7 +790,7 @@ mod tests { fn splice_rejects_top_level_inline_creative_opportunities_table() { let existing = "creative_opportunities = { gam_network_id = \"111\" }\n"; - let error = splice_creative_slots(existing, Some("222"), &header_rendered()) + let error = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect_err("should refuse a top-level inline table"); assert!( @@ -708,6 +799,126 @@ mod tests { ); } + /// Section keys for a templated run: network id plus the section policy. + fn template_keys<'a>( + network_id: &'a str, + root: &'a str, + segment: usize, + ) -> CreativeSectionKeys<'a> { + CreativeSectionKeys { + network_id: Some(network_id), + section_root: Some(root), + section_segment: Some(segment), + } + } + + #[test] + fn splice_inserts_section_policy_keys_a_config_does_not_have_yet() { + // The whole point of `upsert`: every config predating templating lacks + // these keys, so a replace-only writer could never add them. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "inserting must not disturb later sections" + ); + } + + #[test] + fn splice_replaces_section_policy_keys_that_are_already_present() { + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\ + section_root = \"old\"\nsection_segment = 2\n"; + + let out = splice_creative_slots( + existing, + &template_keys("111", "homepage", 1), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(1)); + assert_eq!( + out.matches("section_root").count(), + 1, + "the key must be replaced, not duplicated" + ); + } + + #[test] + fn splice_omits_section_policy_when_no_slot_needs_it() { + // `section_root`/`section_segment` are `deny_unknown_fields` additions: + // writing them into a config that does not need them would make it + // unloadable by an older binary for no benefit. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.contains("section_root") && !out.contains("section_segment"), + "an untemplated run must not add rollback-fatal keys, got:\n{out}" + ); + } + + #[test] + fn splice_writes_section_policy_into_a_freshly_created_section() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + } + + #[test] + fn upsert_keeps_an_inserted_key_inside_the_section_scalar_block() { + // Appending at the end of the section would land the key after a + // subtable, where TOML reads it as part of that subtable instead. + let document = "[creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [[creative_opportunities.slot]]\nid = \"a\"\n\ + page_patterns = [\"/\"]\nformats = [{ width = 1, height = 1 }]\n"; + + let out = upsert_key_in_section( + document, + "creative_opportunities", + "section_root", + "section_root = \"homepage\"", + ) + .expect("should insert"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["section_root"].as_str(), + Some("homepage"), + "the key must belong to the section, not the slot subtable" + ); + } + #[test] fn splice_refuses_fresh_section_without_a_network_id() { // Reachable whenever the scraped unit path has no all-digit leading @@ -716,8 +927,12 @@ mod tests { // route to the startup error router once pushed. let existing = "[publisher]\ndomain = \"x\"\n"; - let error = splice_creative_slots(existing, None, &header_rendered()) - .expect_err("should refuse to create a section with no network id"); + let error = splice_creative_slots( + existing, + &CreativeSectionKeys::default(), + &header_rendered(), + ) + .expect_err("should refuse to create a section with no network id"); assert!( format!("{error:?}").contains("without a GAM network id"), @@ -729,7 +944,7 @@ mod tests { fn splice_appends_section_when_config_has_none() { let existing = "[publisher]\ndomain = \"x\"\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should append a fresh section"); let value = toml::from_str::(&out).expect("appended config is valid TOML"); @@ -771,7 +986,7 @@ mod tests { false, ); - let out = splice_creative_slots(existing, Some("111"), &render_slots(&merged)) + let out = splice_creative_slots(existing, &network_keys("111"), &render_slots(&merged)) .expect("should splice"); let value = toml::from_str::(&out).expect("spliced config is valid TOML"); @@ -803,7 +1018,7 @@ mod tests { let existing = "[creative_opportunities]\r\ngam_network_id = \"111\"\r\n\r\n\ [auction]\r\nenabled = true\r\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should splice"); assert!( @@ -846,7 +1061,7 @@ mod tests { // Config with no [creative_opportunities] at all — generate should append it. let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should splice"); let value = toml::from_str::(&out).expect("valid TOML"); @@ -872,14 +1087,14 @@ mod tests { // header comment; it must keep exactly one copy, not append another. let first = splice_creative_slots( "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n", - Some("222"), + &network_keys("222"), &header_rendered(), ) .expect("first splice"); - let second = - splice_creative_slots(&first, Some("222"), &header_rendered()).expect("second splice"); - let third = - splice_creative_slots(&second, Some("222"), &header_rendered()).expect("third splice"); + let second = splice_creative_slots(&first, &network_keys("222"), &header_rendered()) + .expect("second splice"); + let third = splice_creative_slots(&second, &network_keys("222"), &header_rendered()) + .expect("third splice"); assert_eq!( third @@ -899,7 +1114,7 @@ mod tests { let existing = "[creative_opportunities] # ad templates\ngam_network_id = \"111\"\n\n\ [auction] # flags\nenabled = true\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should splice"); assert_eq!( @@ -931,7 +1146,7 @@ mod tests { let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should splice"); let value = toml::from_str::(&out).expect("valid TOML"); @@ -958,7 +1173,7 @@ mod tests { slot = [{ id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] }]\n\n\ [auction]\nenabled = true\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should replace inline slot array"); let value = toml::from_str::(&out).expect("spliced config should be valid"); @@ -980,7 +1195,7 @@ mod tests { gam_network_id = \"111\"\n\ slot = { \"0\" = { id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] } }\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should replace inline slot map"); let value = toml::from_str::(&out).expect("spliced config should be valid"); From 6722e199675b7e4e6ca69f82712239c4973a80b7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:30:58 +0530 Subject: [PATCH 260/395] Crawl site sections in ad-template generate and write inferred templates Connects the crawl, evidence, inference and writer pieces: a bare `ts audit ad-templates generate ` now samples the site's sections, reconciles each slot across them, infers a `{section}` ad-unit template where the evidence proves one, and writes the section policy alongside the slots. The flow is collect root, plan the crawl from its links and sitemap, walk the planned pages on one browser, fold each into the evidence table, infer, then merge, render, splice and validate as before. Page patterns now come from the sections a slot was actually seen on, so a slot scraped from one article serves its whole section instead of that single URL. Failure handling follows what the evidence can support. A page that will not collect is reported and skipped, because one blocked page should not discard the sections that worked. But if more than a quarter of crawled pages yield no slots the run refuses outright: that is the signature of bot protection serving challenge interstitials, and writing from it would silently narrow the operator's slot set. Pages disagreeing about the GAM network id is likewise a refusal rather than a guess. A run that templates prints the deploy-ordering contract, because the config it just wrote is not rollback-safe: `section_root` and `section_segment` are `deny_unknown_fields` additions, so an older binary rejects the whole config and serves an error on every route. `--max-pages` and `--max-sections` bound the crawl; `--max-pages 1` restores single-page behavior exactly, and an explicit `--page-pattern` still applies to every slot and skips pattern inference. `run_update_slots` takes a request struct, since a nine-argument signature could not absorb the crawl bounds. Removes `default_page_pattern`, superseded by section-derived patterns, and narrows the single-page `merge_slots` path to test scaffolding. --- .../src/commands/audit/generate/collector.rs | 8 - .../src/commands/audit/generate/crawl_plan.rs | 10 +- .../src/commands/audit/generate/evidence.rs | 5 - .../src/commands/audit/generate/mod.rs | 675 ++++++++++++++---- .../commands/audit/generate/page_patterns.rs | 5 - .../src/commands/audit/generate/slot_toml.rs | 50 ++ .../commands/audit/generate/unit_template.rs | 5 - .../src/commands/audit/mod.rs | 37 +- 8 files changed, 635 insertions(+), 160 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs index 625ac660e..e11cf2634 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs @@ -14,10 +14,6 @@ pub(crate) type PageSink<'a> = #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum ControlFlow { /// Collect the next target. - #[allow( - dead_code, - reason = "constructed by run_update_slots once it orchestrates the crawl" - )] Continue, /// Stop the crawl without an error (budget reached, challenge rate exceeded). Stop, @@ -50,10 +46,6 @@ pub(crate) trait AuditCollector { /// /// Returns an error when `on_page` does, or when the session itself cannot /// be established. Individual page failures are delivered to `on_page`. - #[allow( - dead_code, - reason = "called by run_update_slots once it orchestrates the crawl" - )] fn collect_pages( &self, targets: &[Url], diff --git a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs index 5b959dc4f..4796e91dd 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs @@ -14,10 +14,6 @@ //! - **Sitemap entries** give a real *article* per section (`/news/story-abc`), //! which is where in-content slots live, and reveal sections hidden behind a //! navigation overflow menu. -#![allow( - dead_code, - reason = "planner is exercised by tests until run_update_slots orchestrates the crawl" -)] use std::collections::BTreeMap; @@ -58,11 +54,11 @@ const NON_PAGE_EXTENSIONS: &[&str] = &[ /// Bounds on how much of a site a single run will load. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) struct CrawlBudget { +pub(crate) struct CrawlBudget { /// Maximum number of sections to sample. - pub(super) max_sections: usize, + pub(crate) max_sections: usize, /// Maximum number of pages to load in total, including the root. - pub(super) max_pages: usize, + pub(crate) max_pages: usize, } impl Default for CrawlBudget { diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs index e1b1bf81b..bfa004b94 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -19,11 +19,6 @@ //! div ids carry per-render framework hashes and would otherwise look like a new //! slot on every page. -#![allow( - dead_code, - reason = "table is exercised by tests until run_update_slots orchestrates the crawl" -)] - use std::collections::{BTreeMap, BTreeSet}; use super::gpt_slots::DiscoveredSlots; diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 8c1e793ee..4584fd9ba 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -22,14 +22,15 @@ use url::Url; use crate::commands::audit::generate::collector::AuditCollector; use crate::commands::audit::generate::slot_toml::{ - merge_slots, render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, - toml_string, + render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, toml_string, }; use crate::commands::config::init::EXAMPLE_CONFIG; use crate::error::{CliResult, cli_error, report_error}; use analyzer::{analyze_collected_page, extract_gtm_container_id}; +pub(crate) use crawl_plan::CrawlBudget; + /// Writes `contents` to `path` atomically: a same-directory temp file is /// written and fsynced, then renamed over the target, then the directory entry /// is fsynced. @@ -488,70 +489,103 @@ fn render_discovered_slots(target_url: &Url, slots: &gpt_slots::DiscoveredSlots) /// Returns an error when the config cannot be read, the page cannot be /// collected, no slots are discovered, or the config has no /// `[creative_opportunities]` section to update. -#[allow(clippy::too_many_arguments, reason = "cohesive one-shot command entry")] +/// Everything one `ts audit ad-templates generate` invocation needs. +pub(crate) struct UpdateSlotsRequest<'a> { + /// Page URL to start from; also bounds the crawl to its origin. + pub(crate) url: &'a str, + /// Operator config to rewrite in place. + pub(crate) config_path: &'a Path, + /// The config's current `[creative_opportunities]`, when it has one. + pub(crate) existing_creative: Option<&'a CreativeOpportunitiesConfig>, + /// Explicit `--page-pattern` values. When non-empty these apply to every + /// slot and pattern inference is skipped entirely. + pub(crate) page_patterns: &'a [String], + /// Replace existing slots rather than merging into them. + pub(crate) replace: bool, + /// Cookies to carry into the crawl. + pub(crate) cookies: &'a [(String, String)], + /// Print the candidate instead of writing it. + pub(crate) dry_run: bool, + /// Crawl bounds. + pub(crate) budget: crawl_plan::CrawlBudget, +} + +/// Share of crawled pages that may yield no slots before the run is refused. +/// +/// A bot-protection challenge serves an interstitial that loads fine and +/// contains no ad stack, so it looks like a page with no slots. Writing a config +/// from a crawl that was mostly challenges would silently narrow the operator's +/// slot set; refusing is the safer failure. +const MAX_EMPTY_PAGE_SHARE: f64 = 0.25; + +/// Runs `ts audit ad-templates generate`: crawl the site's sections, reconcile +/// what each slot looked like across them, infer a `{section}` ad-unit template +/// where the evidence proves one, and rewrite the config's slot array in place. +/// +/// # Errors +/// +/// Returns an error when the config cannot be read, the root page cannot be +/// collected, no slots are discovered, too many pages came back empty, the +/// pages disagree about the GAM network id, or the resulting config would not +/// load. pub(crate) fn run_update_slots( - url: &str, - config_path: &Path, - existing_creative: Option<&CreativeOpportunitiesConfig>, - page_patterns: &[String], - replace: bool, - cookies: &[(String, String)], - dry_run: bool, + request: &UpdateSlotsRequest<'_>, collector: &dyn AuditCollector, out: &mut dyn Write, ) -> CliResult<()> { - let target_url = parse_audit_url(url)?; - let existing = fs::read_to_string(config_path).map_err(|error| { + let target_url = parse_audit_url(request.url)?; + let existing = fs::read_to_string(request.config_path).map_err(|error| { report_error(format!( "failed to read config {}: {error}", - config_path.display() + request.config_path.display() )) })?; - let collected = collector.collect_page(&target_url, cookies)?; - let artifact = analyze_collected_page(&collected)?; - let page_has_prebid = artifact - .detected_integrations - .iter() - .any(|integration| integration.id == "prebid"); - let discovered = gpt_slots::discover_gpt_slots( - &collected.gpt_slots, - &collected.network_requests, - page_has_prebid, - ); - if discovered.slots.is_empty() { - return cli_error("no ad-template slots were discovered on the page"); - } + let root = collector.collect_page(&target_url, request.cookies)?; + let root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + fold_collected(&mut table, &root_url, &root)?; - // Patterns for slots seen on this run: the `--page-pattern` values, or the - // audited path when none are given (preserving single-page behavior). The - // default uses the recorded post-redirect URL so it matches the page that - // was actually audited, falling back to the requested URL when the - // recorded final URL is invalid. - let run_patterns: Vec = if page_patterns.is_empty() { - let audited_url = collected.final_url().unwrap_or_else(|_| target_url.clone()); - vec![default_page_pattern(&audited_url)] - } else { - page_patterns.to_vec() - }; - // Reject a pattern the runtime cannot compile before it reaches the file: - // a persisted invalid glob either fails the next config load or is silently - // dropped at pattern-compile time, leaving the slot matching fewer pages - // than the config claims. - validate_page_patterns(&run_patterns)?; + // One page per section is enough: ad slots repeat per section, so the crawl + // is sized by the publisher's taxonomy rather than its catalogue. + let plan = crawl_plan::plan_crawl(&root_url, &root.links, &root.sitemap_locs, request.budget); + notes.extend(plan.notes.iter().cloned()); + crawl_sections(collector, &plan, request.cookies, &mut table, &mut notes)?; - let merged = merge_slots(existing_creative, &discovered, &run_patterns, replace); + if table.is_empty() { + return cli_error("no ad-template slots were discovered on any crawled page"); + } + guard_challenge_rate(&table)?; + + let discovered_network_id = table.network_id()?; let network_id = resolve_network_id( - existing_creative, - discovered.gam_network_id.as_deref(), - replace, + request.existing_creative, + discovered_network_id.as_deref(), + request.replace, ); + + // Templating needs a network id to bind `{network_id}` against; without one + // every path stays literal. + let inference = network_id + .as_deref() + .map(|id| unit_template::infer_unit_templates(&table, id)); + if let Some(outcome) = &inference { + notes.extend(outcome.diagnostics.iter().cloned()); + } + let policy = inference + .as_ref() + .and_then(|outcome| outcome.policy.clone()); + + let slots = build_render_slots(&table, inference.as_ref(), policy.as_ref(), request)?; + let merged = slot_toml::merge_render_slots(request.existing_creative, slots, request.replace); let rendered_slots = render_slots(&merged); let updated = splice_creative_slots( &existing, &slot_toml::CreativeSectionKeys { network_id: network_id.as_deref(), - ..slot_toml::CreativeSectionKeys::default() + section_root: policy.as_ref().map(|policy| policy.section_root.as_str()), + section_segment: policy.as_ref().map(|policy| policy.section_segment), }, &rendered_slots, )?; @@ -560,31 +594,165 @@ pub(crate) fn run_update_slots( // candidate has to clear the runtime's own load path before it can replace // the operator's file. This runs on the dry-run path too — otherwise "the // preview looked fine" would not be evidence that the config loads. - for warning in validate::check_candidate(&updated, &existing)? { - writeln!(out, "warning: {warning}") + notes.extend(validate::check_candidate(&updated, &existing)?); + + for note in ¬es { + writeln!(out, "note: {note}") .map_err(|error| report_error(format!("failed to write command output: {error}")))?; } + if policy.is_some() { + writeln!( + out, + "note: this config now uses a {{section}} ad-unit template. Deploy a \ + template-aware binary BEFORE pushing it, and do not roll that binary \ + back while this config is live — an older binary rejects the whole \ + config and serves an error on every route." + ) + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } - if dry_run { + if request.dry_run { writeln!(out, "{updated}") .map_err(|error| report_error(format!("failed to write preview: {error}")))?; return Ok(()); } - write_file_atomically(config_path, &updated).map_err(|error| { + write_file_atomically(request.config_path, &updated).map_err(|error| { report_error(format!( "failed to write config {}: {error}", - config_path.display() + request.config_path.display() )) })?; writeln!( out, - "Wrote {} slot(s) to {} ({} discovered this run)", + "Wrote {} slot(s) to {} ({} slot(s) seen across {} page(s))", merged.len(), - config_path.display(), - discovered.slots.len(), + request.config_path.display(), + table.slot_count(), + table.pages().len(), ) .map_err(|error| report_error(format!("failed to write command output: {error}"))) } + +/// Discovers a collected page's slots and folds them into `table`. +fn fold_collected( + table: &mut evidence::EvidenceTable, + url: &Url, + collected: &collector::CollectedPage, +) -> CliResult<()> { + let artifact = analyze_collected_page(collected)?; + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let discovered = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + table.fold_page(url.path(), &discovered); + Ok(()) +} + +/// Walks the planned section pages, folding each into `table`. +/// +/// A page that fails to collect is recorded as a note rather than aborting: on a +/// multi-section crawl one blocked or slow page should not discard the sections +/// that did work. The empty-page guard afterwards catches the case where enough +/// of them failed that the result is untrustworthy. +fn crawl_sections( + collector: &dyn AuditCollector, + plan: &crawl_plan::CrawlPlan, + cookies: &[(String, String)], + table: &mut evidence::EvidenceTable, + notes: &mut Vec, +) -> CliResult<()> { + let targets = plan.targets(); + if targets.is_empty() { + notes.push( + "no additional site sections were discovered, so only the requested page was \ + audited; pass explicit --page-pattern values or more URLs to widen coverage" + .to_string(), + ); + return Ok(()); + } + + let mut fold_error = None; + collector.collect_pages(&targets, cookies, &mut |url, collected| { + match collected { + Ok(page) => { + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + if let Err(error) = fold_collected(table, &final_url, &page) { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => notes.push(format!("skipped `{url}`: {error}")), + } + Ok(collector::ControlFlow::Continue) + })?; + match fold_error { + Some(error) => Err(error), + None => Ok(()), + } +} + +/// Refuses a crawl where too many pages produced no slots. +fn guard_challenge_rate(table: &evidence::EvidenceTable) -> CliResult<()> { + let total = table.pages().len(); + let empty = table.empty_pages().len(); + if total == 0 || (empty as f64) <= (total as f64) * MAX_EMPTY_PAGE_SHARE { + return Ok(()); + } + let blocked: Vec<&str> = table.empty_pages().iter().map(String::as_str).collect(); + cli_error(format!( + "{empty} of {total} crawled page(s) produced no ad slots ({}), which usually means \ + bot protection served a challenge instead of the real page. Refusing to write a \ + config from partial evidence; re-run with a valid --cookie for the origin", + blocked.join(", ") + )) +} + +/// Turns the evidence table into slots ready to render. +fn build_render_slots( + table: &evidence::EvidenceTable, + inference: Option<&unit_template::InferenceOutcome>, + policy: Option<&unit_template::SectionPolicy>, + request: &UpdateSlotsRequest<'_>, +) -> CliResult> { + // Explicit `--page-pattern` values are an operator override: they apply to + // every slot and disable inference from observed paths entirely. + let explicit = !request.page_patterns.is_empty(); + if explicit { + validate_page_patterns(request.page_patterns)?; + } + let section_segment = policy.map_or(0, |policy| policy.section_segment); + + let mut slots = Vec::with_capacity(table.slot_count()); + for slot in table.slots() { + let patterns = if explicit { + request.page_patterns.to_vec() + } else { + let derived = page_patterns::patterns_for_paths(slot.paths(), section_segment); + validate_page_patterns(&derived)?; + derived + }; + let unit_path = match inference.and_then(|outcome| outcome.decision(&slot.div_id)) { + Some(unit_template::SlotDecision::Template(template)) => Some(template.clone()), + Some(unit_template::SlotDecision::Literal(path)) => Some(path.clone()), + // Refused: write the slot without a path rather than a wrong one. + Some(unit_template::SlotDecision::Refuse { .. }) | None => None, + }; + slots.push(slot_toml::RenderSlot::from_evidence( + &slot.id, + &slot.div_id, + unit_path, + slot.formats.iter().copied(), + patterns, + slot.has_prebid, + )); + } + Ok(slots) +} /// Rejects any page pattern the runtime's glob compiler would not accept. /// /// Uses [`compile_page_pattern`] so the accepted set is exactly what @@ -609,16 +777,6 @@ fn validate_page_patterns(patterns: &[String]) -> CliResult<()> { )) } -/// The default page pattern for a scraped URL: its path, or `/` for the root. -fn default_page_pattern(target_url: &Url) -> String { - let path = target_url.path(); - if path.is_empty() { - "/".to_string() - } else { - path.to_string() - } -} - #[cfg(test)] mod tests { use std::cell::Cell; @@ -657,6 +815,58 @@ mod tests { } } + /// A collector serving a distinct page per URL, recording the crawl order. + struct SiteCollector { + pages: std::collections::HashMap, + visited: std::cell::RefCell>, + } + + impl SiteCollector { + fn new(pages: Vec<(&str, CollectedPage)>) -> Self { + Self { + pages: pages + .into_iter() + .map(|(url, page)| (url.to_string(), page)) + .collect(), + visited: std::cell::RefCell::new(Vec::new()), + } + } + } + + impl AuditCollector for SiteCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + self.visited.borrow_mut().push(target_url.to_string()); + self.pages + .get(target_url.as_str()) + .cloned() + .ok_or_else(|| report_error(format!("no fake page for {target_url}"))) + } + } + + /// Builds a page carrying one GPT slot plus same-origin nav links. + fn site_page(url: &str, unit_path: &str, nav_paths: &[&str]) -> CollectedPage { + let mut page = collected_page(); + page.requested_url = url.to_string(); + page.final_url = url.to_string(); + page.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: unit_path.to_string(), + div_id: "ad-header-0".to_string(), + sizes: vec![(728, 90)], + }]; + page.links = nav_paths + .iter() + .map(|path| collector::CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + }) + .collect(); + page + } + fn collected_page() -> CollectedPage { CollectedPage { requested_url: "https://publisher.example/page".to_string(), @@ -1042,13 +1252,16 @@ mod tests { let mut out = Vec::new(); run_update_slots( - "https://publisher.example/", - &config_path, - None, - &[], - false, - &[], - false, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1056,10 +1269,19 @@ mod tests { let written = fs::read_to_string(&config_path).expect("should read config"); let value = toml::from_str::(&written).expect("valid TOML"); + let patterns: Vec<&str> = value["creative_opportunities"]["slot"][0]["page_patterns"] + .as_array() + .expect("page_patterns array") + .iter() + .map(|entry| entry.as_str().expect("pattern string")) + .collect(); + // Patterns come from the post-redirect path: had the requested `/` been + // used, this would be `["/"]`. They now cover the whole section rather + // than only the one article that happened to be scraped. assert_eq!( - value["creative_opportunities"]["slot"][0]["page_patterns"][0].as_str(), - Some("/news/story"), - "default pattern should use the post-redirect path, not the requested one" + patterns, + ["/news", "/news/*"], + "should derive section patterns from the post-redirect path" ); } @@ -1073,13 +1295,16 @@ mod tests { let mut out = Vec::new(); let error = run_update_slots( - "https://publisher.example/", - &config_path, - None, - &["[".to_string()], - false, - &[], - false, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["[".to_string()], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1111,13 +1336,16 @@ mod tests { let mut out = Vec::new(); run_update_slots( - "https://publisher.example/", - &config_path, - None, - &["/20**".to_string()], - false, - &[], - false, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["/20**".to_string()], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1144,13 +1372,16 @@ mod tests { let mut out = Vec::new(); run_update_slots( - "https://publisher.example/", - &config_path, - None, - &[], - false, - &[], - false, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1193,6 +1424,210 @@ mod tests { ) } + #[test] + fn a_crawl_writes_a_section_template_and_per_section_patterns() { + // The end-to-end payoff: crawl sections, reconcile the slot across them, + // infer `{section}`, and write a config the runtime loads. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&config_path, loadable_config()).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/site/news", + &nav, + ), + ), + ( + "https://publisher.example/deals", + site_page( + "https://publisher.example/deals", + "/123456789/site/deals", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, + &collector, + &mut out, + ) + .expect("should crawl and update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "the unvisited-section fallback should come from the root page" + ); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + let slot = &creative["slot"][0]; + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/{network_id}/site/{section}"), + "the varying segment should become a template" + ); + let patterns: Vec<&str> = slot["page_patterns"] + .as_array() + .expect("patterns array") + .iter() + .map(|entry| entry.as_str().expect("pattern")) + .collect(); + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "each witnessed section should contribute both halves of its pair" + ); + + // The whole point of the gate: what was written must actually load. + trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + + let report = String::from_utf8(out).expect("utf8 output"); + assert!( + report.contains("Deploy a template-aware binary BEFORE pushing"), + "a templated config must warn about the rollback contract, got:\n{report}" + ); + } + + #[test] + fn a_crawl_refuses_when_most_pages_are_challenged() { + // Bot protection serves an interstitial that loads fine and has no ad + // stack, so it looks like a page with no slots. Writing from that would + // silently narrow the operator's slot set. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let mut blocked_news = site_page("https://publisher.example/news", "/123456789/x", &nav); + blocked_news.gpt_slots.clear(); + let mut blocked_deals = site_page("https://publisher.example/deals", "/123456789/x", &nav); + blocked_deals.gpt_slots.clear(); + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ("https://publisher.example/news", blocked_news), + ("https://publisher.example/deals", blocked_deals), + ]); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, + &collector, + &mut out, + ) + .expect_err("a mostly-challenged crawl should refuse"); + + assert!( + format!("{error:?}").contains("bot protection"), + "the error should name the likely cause, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a refused run must leave the config untouched" + ); + } + + #[test] + fn max_pages_one_restores_single_page_behavior() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&config_path, loadable_config()).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + )]); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget { + max_sections: 8, + max_pages: 1, + }, + }, + &collector, + &mut out, + ) + .expect("should update from the single page"); + + assert_eq!( + collector.visited.borrow().len(), + 1, + "max_pages = 1 must not crawl beyond the requested page" + ); + let written = fs::read_to_string(&config_path).expect("read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert!( + value["creative_opportunities"] + .get("section_root") + .is_none(), + "one page cannot witness a section, so no rollback-fatal key may be written" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["gam_unit_path"].as_str(), + Some("/123456789/site/homepage"), + "a single page keeps the literal path" + ); + } + #[test] fn generated_config_loads_through_the_runtime_settings_path() { // The end-to-end contract: whatever `generate` writes must survive the @@ -1208,13 +1643,16 @@ mod tests { let mut out = Vec::new(); run_update_slots( - "https://publisher.example/", - &config_path, - None, - &[], - false, - &[], - false, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1301,13 +1739,16 @@ mod tests { let mut out = Vec::new(); run_update_slots( - "https://publisher.example/", - &loaded.app_config_path, - loaded.settings.creative_opportunities.as_ref(), - &[], - false, - &[], - true, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &loaded.app_config_path, + existing_creative: loaded.settings.creative_opportunities.as_ref(), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1329,16 +1770,4 @@ mod tests { }, ); } - - #[test] - fn default_page_pattern_uses_path_or_root() { - assert_eq!( - default_page_pattern(&Url::parse("https://x/news/story").expect("url")), - "/news/story" - ); - assert_eq!( - default_page_pattern(&Url::parse("https://x/").expect("url")), - "/" - ); - } } diff --git a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs index 70b18a6ae..5c8c3fd27 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs @@ -10,11 +10,6 @@ //! **not** the bare `/news` landing page. Emitting only the star form silently //! drops the landing page from the slot. -#![allow( - dead_code, - reason = "expansion is exercised by tests until run_update_slots orchestrates the crawl" -)] - use std::collections::BTreeSet; /// The root pattern, matching only the site root. diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index a0fc51da7..9d01d8dc5 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -10,6 +10,7 @@ use trusted_server_core::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunitySlot, }; +#[cfg(test)] use crate::commands::audit::generate::gpt_slots; use crate::error::{CliResult, cli_error, report_error}; @@ -41,6 +42,11 @@ impl RenderSlot { .to_string() } + /// Builds a slot from one page's discovery. + /// + /// Superseded in production by [`RenderSlot::from_evidence`], which reads + /// cross-page evidence; retained as test scaffolding for the merge cases. + #[cfg(test)] fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { Self { id: slot.id.clone(), @@ -59,6 +65,35 @@ impl RenderSlot { } } + /// Builds a slot from cross-page evidence and the inferred unit path. + /// + /// `gam_unit_path` is `None` when inference refused to represent the slot; + /// the slot is still written so its div and formats are not lost, and the + /// runtime falls back to the default `//` path. + pub(super) fn from_evidence( + id: &str, + div_id: &str, + gam_unit_path: Option, + formats: impl IntoIterator, + page_patterns: Vec, + has_prebid: bool, + ) -> Self { + Self { + id: id.to_string(), + div_id: Some(div_id.to_string()), + gam_unit_path, + page_patterns, + formats: formats + .into_iter() + .map(|(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: has_prebid.then(BTreeMap::new), + } + } + fn from_existing(slot: &CreativeOpportunitySlot) -> Self { Self { id: slot.id.clone(), @@ -109,6 +144,7 @@ fn media_type_label(media_type: &MediaType) -> Option<&'static str> { /// - Otherwise existing slots are preserved (covering other pages / hand-tuned /// fields); a slot re-seen this run has `run_patterns` unioned into its /// `page_patterns`; slots seen only this run are appended. +#[cfg(test)] pub(super) fn merge_slots( existing: Option<&CreativeOpportunitiesConfig>, discovered: &gpt_slots::DiscoveredSlots, @@ -120,7 +156,21 @@ pub(super) fn merge_slots( .iter() .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) .collect(); + merge_render_slots(existing, discovered_slots, replace) +} +/// Merges already-built slots into the existing set. +/// +/// Same reconciliation as [`merge_slots`], but the caller supplies the slots — +/// the crawl path builds them from cross-page evidence rather than from one +/// page's discoveries. A slot re-seen this run keeps its configured fields and +/// gains this run's patterns; a genuinely new slot is appended with a +/// non-colliding id. +pub(super) fn merge_render_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + replace: bool, +) -> Vec { let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); if replace || existing_slots.is_empty() { return discovered_slots; diff --git a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs index 790a89859..d4874b2b2 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs @@ -25,11 +25,6 @@ //! [`derive_section`] against every observation. A template that does not //! reproduce what the live page actually requested is downgraded, not written. -#![allow( - dead_code, - reason = "inference is exercised by tests until run_update_slots orchestrates the crawl" -)] - use std::collections::{BTreeMap, BTreeSet}; use trusted_server_core::creative_opportunities::{CreativeOpportunitySlot, derive_section}; diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index e024eb8b1..120ac86be 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -137,6 +137,26 @@ pub(crate) struct AuditAdTemplatesGenerateArgs { /// cookie) so the origin serves the real page instead of a challenge. #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] pub cookies: Vec<(String, String)>, + /// Maximum site sections to sample. Each contributes a landing page and an + /// article, so this bounds how much of the publisher's taxonomy is covered. + #[arg(long, default_value_t = 8)] + pub max_sections: usize, + /// Maximum pages to load in total, including the requested page. + /// + /// Set to 1 to restore single-page behavior: no crawl, no section + /// discovery, and the audited path as the only page pattern. + #[arg(long, default_value_t = 17)] + pub max_pages: usize, +} + +impl AuditAdTemplatesGenerateArgs { + /// The crawl bounds these arguments describe. + pub(crate) fn budget(&self) -> generate::CrawlBudget { + generate::CrawlBudget { + max_sections: self.max_sections, + max_pages: self.max_pages, + } + } } /// Arguments for `ts audit ad-templates verify ...`. @@ -190,13 +210,16 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { let stdout = std::io::stdout(); let mut out = stdout.lock(); generate::run_update_slots( - gen_args.url.as_str(), - &loaded.app_config_path, - loaded.settings.creative_opportunities.as_ref(), - &gen_args.page_patterns, - gen_args.replace, - &gen_args.cookies, - gen_args.dry_run, + &generate::UpdateSlotsRequest { + url: gen_args.url.as_str(), + config_path: &loaded.app_config_path, + existing_creative: loaded.settings.creative_opportunities.as_ref(), + page_patterns: &gen_args.page_patterns, + replace: gen_args.replace, + cookies: &gen_args.cookies, + dry_run: gen_args.dry_run, + budget: gen_args.budget(), + }, &collector, &mut out, ) From d9133f2820e0784b083a6108c6eba7e9ae3655f7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:37:07 +0530 Subject: [PATCH 261/395] Add device-profile cross-checking to ad-template generate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishers routinely serve a different GAM ad unit per device (`/network/desktop/news` against `/network/mobile/news`). A single-profile crawl cannot see that: it infers a template that is correct for the profile it used and silently wrong for every other impression, with nothing in the data to say so. This was the one unmitigated risk in the inference design. `--profiles desktop,mobile` walks every planned page once per profile, each with its own viewport and user agent, folding all of it into one evidence table. The user agent matters as much as the viewport here — ad stacks branch on it, so emulating size alone can still return desktop ad units on a phone-sized page. No new refusal logic was needed. Two profiles disagreeing produce two ad-unit paths for a single page, which is already the structural refusal inference applies to a unit that varies by something the request path cannot derive. The slot is still written, with its div and formats intact, but with no `gam_unit_path`: no path at all is better than one that is wrong on mobile, and the runtime falls back to the default unit rather than bidding on a unit that does not exist. Desktop-only stays the default, so the extra crawl is opt-in. --- .../audit/generate/browser_collector.rs | 126 +++++++++++++++-- .../src/commands/audit/generate/mod.rs | 132 ++++++++++++++++-- .../src/commands/audit/mod.rs | 53 ++++++- 3 files changed, 284 insertions(+), 27 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index b1c504cd5..eead2d49a 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -4,6 +4,7 @@ use std::time::Duration; use chromiumoxide::ArcHttpRequest; use chromiumoxide::browser::{Browser, BrowserConfig}; use chromiumoxide::cdp::browser_protocol::network::CookieParam; +use chromiumoxide::handler::viewport::Viewport; use futures::StreamExt as _; use serde::Deserialize; use tempfile::TempDir; @@ -32,8 +33,98 @@ const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; const RESOURCE_TIMING_BUFFER_WARNING: &str = "browser resource timing buffer reached its default size; some network assets may be missing"; -#[derive(Default)] -pub(crate) struct BrowserAuditCollector; +/// A device the crawl can emulate. +/// +/// Publishers routinely serve different GAM ad units per device +/// (`/network/desktop/news` vs `/network/mobile/news`). A single-profile crawl +/// cannot see that, so it would infer a template that is right for the profile +/// it used and silently wrong for every other impression. Crawling twice makes +/// the disagreement visible: the two profiles produce two ad-unit paths for the +/// same page, which template inference already treats as unrepresentable. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DeviceProfile { + /// A desktop viewport with Chrome's own user agent. + Desktop, + /// A phone viewport with touch and a mobile user agent. + Mobile, +} + +impl DeviceProfile { + /// The operator-facing name, matching the `--profiles` value. + pub(crate) fn label(self) -> &'static str { + match self { + Self::Desktop => "desktop", + Self::Mobile => "mobile", + } + } + + /// Parses a `--profiles` value. + /// + /// # Errors + /// + /// Returns an error naming the accepted values when `raw` is not one. + pub(crate) fn parse(raw: &str) -> Result { + match raw.trim().to_ascii_lowercase().as_str() { + "desktop" => Ok(Self::Desktop), + "mobile" => Ok(Self::Mobile), + other => Err(format!( + "unknown device profile `{other}` (expected desktop or mobile)" + )), + } + } + + /// The viewport to emulate. + fn viewport(self) -> Viewport { + match self { + Self::Desktop => Viewport { + width: 1280, + height: 800, + device_scale_factor: Some(1.0), + emulating_mobile: false, + is_landscape: true, + has_touch: false, + }, + Self::Mobile => Viewport { + width: 390, + height: 844, + device_scale_factor: Some(3.0), + emulating_mobile: true, + is_landscape: false, + has_touch: true, + }, + } + } + + /// The user agent override, or `None` to keep Chrome's own. + /// + /// Ad stacks branch on the user agent as well as the viewport, so emulating + /// the viewport alone can still yield desktop ad units on a phone-sized page. + fn user_agent(self) -> Option<&'static str> { + match self { + Self::Desktop => None, + Self::Mobile => Some( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", + ), + } + } +} + +/// Collects pages through a local Chrome, emulating one device profile. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct BrowserAuditCollector { + profile: Option, +} + +impl BrowserAuditCollector { + /// A collector emulating `profile`. + #[must_use] + pub(crate) fn with_profile(profile: DeviceProfile) -> Self { + Self { + profile: Some(profile), + } + } +} impl AuditCollector for BrowserAuditCollector { fn collect_page( @@ -50,11 +141,13 @@ impl AuditCollector for BrowserAuditCollector { )) })?; + let profile = self.profile; runtime.block_on(async { let mut collected = None; with_browser( std::slice::from_ref(target_url), cookies, + profile, &mut |_, result| { collected = Some(result); Ok(ControlFlow::Stop) @@ -83,7 +176,7 @@ impl AuditCollector for BrowserAuditCollector { )) })?; - runtime.block_on(with_browser(targets, cookies, on_page)) + runtime.block_on(with_browser(targets, cookies, self.profile, on_page)) } } @@ -98,6 +191,7 @@ impl AuditCollector for BrowserAuditCollector { async fn with_browser( targets: &[Url], cookies: &[(String, String)], + profile: Option, sink: PageSink<'_>, ) -> CliResult<()> { let chrome_executable = find_browser_executable()?; @@ -110,17 +204,27 @@ async fn with_browser( // cookies and writes what it scrapes into the operator's config, so a // certificate-invalid impersonator could both harvest the session and seed // the config with slots of its choosing. Validate certificates. - let config = BrowserConfig::builder() + let mut builder = BrowserConfig::builder() .chrome_executable(chrome_executable) .user_data_dir(user_data_dir.path()) .new_headless_mode() - .respect_https_errors() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Chromium configuration for audit: {error}" - )) - })?; + .respect_https_errors(); + if let Some(profile) = profile { + let viewport = profile.viewport(); + builder = builder + .window_size(viewport.width, viewport.height) + .viewport(viewport); + if let Some(user_agent) = profile.user_agent() { + // Ad stacks branch on the user agent as well as the viewport, so + // emulating size alone can still return desktop ad units. + builder = builder.arg(format!("--user-agent={user_agent}")); + } + } + let config = builder.build().map_err(|error| { + report_error(format!( + "failed to build Chromium configuration for audit: {error}" + )) + })?; let (mut browser, mut handler) = Browser::launch(config).await.map_err(|error| { report_error(format!( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 4584fd9ba..d31030ec7 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -29,6 +29,7 @@ use crate::error::{CliResult, cli_error, report_error}; use analyzer::{analyze_collected_page, extract_gtm_container_id}; +pub(crate) use browser_collector::DeviceProfile; pub(crate) use crawl_plan::CrawlBudget; /// Writes `contents` to `path` atomically: a same-directory temp file is @@ -530,9 +531,12 @@ const MAX_EMPTY_PAGE_SHARE: f64 = 0.25; /// load. pub(crate) fn run_update_slots( request: &UpdateSlotsRequest<'_>, - collector: &dyn AuditCollector, + collectors: &[(&str, &dyn AuditCollector)], out: &mut dyn Write, ) -> CliResult<()> { + let Some((_, first_collector)) = collectors.first() else { + return cli_error("no device profile was selected to audit with"); + }; let target_url = parse_audit_url(request.url)?; let existing = fs::read_to_string(request.config_path).map_err(|error| { report_error(format!( @@ -541,7 +545,7 @@ pub(crate) fn run_update_slots( )) })?; - let root = collector.collect_page(&target_url, request.cookies)?; + let root = first_collector.collect_page(&target_url, request.cookies)?; let root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); let mut table = evidence::EvidenceTable::default(); let mut notes = Vec::new(); @@ -551,7 +555,31 @@ pub(crate) fn run_update_slots( // is sized by the publisher's taxonomy rather than its catalogue. let plan = crawl_plan::plan_crawl(&root_url, &root.links, &root.sitemap_locs, request.budget); notes.extend(plan.notes.iter().cloned()); - crawl_sections(collector, &plan, request.cookies, &mut table, &mut notes)?; + + // Every profile walks the same pages into the same table. When two profiles + // disagree about a slot's ad-unit path, that shows up as two observations of + // one page, which inference already refuses to represent. + for (index, (label, collector)) in collectors.iter().enumerate() { + if index > 0 { + let repeat = first_collector.collect_page(&root_url, request.cookies); + match repeat { + Ok(page) => fold_collected(&mut table, &root_url, &page)?, + Err(error) => notes.push(format!("skipped `{root_url}` on {label}: {error}")), + } + } + crawl_sections(*collector, &plan, request.cookies, &mut table, &mut notes)?; + } + if collectors.len() > 1 { + notes.push(format!( + "audited {} device profile(s): {}", + collectors.len(), + collectors + .iter() + .map(|(label, _)| *label) + .collect::>() + .join(", ") + )); + } if table.is_empty() { return cli_error("no ad-template slots were discovered on any crawled page"); @@ -1262,7 +1290,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should update slots"); @@ -1305,7 +1333,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect_err("should reject an invalid glob"); @@ -1346,7 +1374,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should accept a runtime-normalisable pattern"); @@ -1382,7 +1410,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should update slots"); @@ -1472,7 +1500,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should crawl and update slots"); @@ -1516,6 +1544,86 @@ mod tests { ); } + #[test] + fn disagreeing_device_profiles_refuse_to_write_a_unit_path() { + // Two profiles serving different ad units for the same page is exactly + // the failure a single-profile crawl cannot see. Writing either path + // would be correct for one device and silently wrong for the other. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&config_path, loadable_config()).expect("should write config"); + + let nav = ["/news"]; + let desktop = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/desktop/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/desktop/news", + &nav, + ), + ), + ]); + let mobile = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/mobile/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/mobile/news", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + ) + .expect("the run should complete and report the conflict"); + + let written = fs::read_to_string(&config_path).expect("read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert!( + creative.get("section_root").is_none(), + "a device split must not produce a section template" + ); + assert!( + creative["slot"][0].get("gam_unit_path").is_none(), + "no ad-unit path is better than one that is wrong on mobile, got:\n{written}" + ); + // What was written must still load. + trusted_server_core::settings::Settings::from_toml(&written) + .expect("a slot without an explicit unit path must still load"); + } + #[test] fn a_crawl_refuses_when_most_pages_are_challenged() { // Bot protection serves an interstitial that loads fine and has no ad @@ -1556,7 +1664,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect_err("a mostly-challenged crawl should refuse"); @@ -1603,7 +1711,7 @@ mod tests { max_pages: 1, }, }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should update from the single page"); @@ -1653,7 +1761,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should update slots"); @@ -1749,7 +1857,7 @@ mod tests { dry_run: true, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should render dry-run update"); diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 120ac86be..b526a4088 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -147,6 +147,15 @@ pub(crate) struct AuditAdTemplatesGenerateArgs { /// discovery, and the audited path as the only page pattern. #[arg(long, default_value_t = 17)] pub max_pages: usize, + /// Device profiles to audit, comma-separated: `desktop`, `mobile`. + /// + /// Defaults to `desktop`. Publishers often serve different GAM ad units per + /// device, which a single-profile crawl cannot see — it would infer a + /// template correct for the profile it used and silently wrong elsewhere. + /// Passing both crawls each page twice and refuses to write an ad-unit path + /// for any slot where the profiles disagree. + #[arg(long, value_delimiter = ',', default_value = "desktop")] + pub profiles: Vec, } impl AuditAdTemplatesGenerateArgs { @@ -157,6 +166,26 @@ impl AuditAdTemplatesGenerateArgs { max_pages: self.max_pages, } } + + /// The device profiles to audit, deduplicated in the order given. + /// + /// # Errors + /// + /// Returns an error when a name is not a known profile, or when none were + /// given. + pub(crate) fn profiles(&self) -> Result, String> { + let mut profiles: Vec = Vec::new(); + for raw in &self.profiles { + let profile = generate::DeviceProfile::parse(raw)?; + if !profiles.contains(&profile) { + profiles.push(profile); + } + } + if profiles.is_empty() { + return Err("--profiles needs at least one of: desktop, mobile".to_string()); + } + Ok(profiles) + } } /// Arguments for `ts audit ad-templates verify ...`. @@ -206,7 +235,23 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { Some(AuditSubcommand::Page(page_args)) => page::run_page(page_args), Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { let loaded = crate::app_config::load_file_settings(&gen_args.config)?; - let collector = generate::browser_collector::BrowserAuditCollector; + let profiles = gen_args.profiles()?; + let collectors: Vec = profiles + .iter() + .map(|profile| { + generate::browser_collector::BrowserAuditCollector::with_profile(*profile) + }) + .collect(); + let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles + .iter() + .zip(collectors.iter()) + .map(|(profile, collector)| { + ( + profile.label(), + collector as &dyn generate::collector::AuditCollector, + ) + }) + .collect(); let stdout = std::io::stdout(); let mut out = stdout.lock(); generate::run_update_slots( @@ -220,7 +265,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { dry_run: gen_args.dry_run, budget: gen_args.budget(), }, - &collector, + &selected, &mut out, ) } @@ -230,7 +275,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { Some(AuditSubcommand::Generate(generate_args)) => { let stdout = std::io::stdout(); let mut out = stdout.lock(); - let collector = generate::browser_collector::BrowserAuditCollector; + let collector = generate::browser_collector::BrowserAuditCollector::default(); generate::run_generate(generate_args, &collector, &mut out) } None => match &args.legacy_url { @@ -239,7 +284,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { .expect("should build generation args when legacy URL is present"); let stdout = std::io::stdout(); let mut out = stdout.lock(); - let collector = generate::browser_collector::BrowserAuditCollector; + let collector = generate::browser_collector::BrowserAuditCollector::default(); generate::run_generate(&generate_args, &collector, &mut out) } None => Err("provide a URL or a subcommand (`page`, `ad-templates`)".to_string()), From 2ea48e097583b4c4c944bfbbdddac4e8756ce088 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:41:14 +0530 Subject: [PATCH 262/395] Document ad-template slot generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ts audit ad-templates generate` had no documentation at all. Cover what the crawl does, what it writes, and the two things an operator cannot discover from the output alone. The first is when the command declines to generalize. A wrong ad-unit template makes a publisher bid against inventory that does not exist, so the command prefers a narrow literal path over a plausible guess, and the table says which situations produce which outcome — including the cases that fail the run outright, such as a crawl where bot protection served mostly challenge pages. The second is deploy ordering. A config carrying `section_root` or `section_segment` is not rollback-safe: a binary predating ad-unit templating rejects those keys, and the rejection fails the whole configuration load rather than just the ad-template section, so every route serves an error. Ship the template-aware binary first, push second, and do not roll back while that config is live. --- docs/guide/cli.md | 129 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/docs/guide/cli.md b/docs/guide/cli.md index e0baac367..b0aeb612b 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -138,6 +138,135 @@ ts audit generate https://publisher.example --force The legacy `ts audit ` form remains a compatibility alias for artifact generation. New automation should use `ts audit generate `. +## Generate ad-template slots from a live site + +`ts audit ad-templates generate ` discovers the publisher's ad slots and +rewrites the `[creative_opportunities]` slot array in `trusted-server.toml` in +place, preserving every other section and comment. + +```bash +ts audit ad-templates generate https://publisher.example/ +``` + +It samples the site rather than a single page. Ad slots repeat per site +section, so the crawl is sized by the publisher's taxonomy — a dozen sections — +not its catalogue: + +1. Load the requested page and read its links and, from `robots.txt`, its + sitemap. +2. Group both into candidate sections, keeping one landing page and one article + per section. +3. Load those pages, recording each slot's div, sizes, and GAM ad-unit path. +4. Reconcile every slot across the pages it appeared on. +5. Infer a `{section}` ad-unit template if the evidence proves one. +6. Verify the result loads, then write it. + +### What it writes + +Given a site whose ad units track the section, the run produces: + +```toml +[creative_opportunities] +gam_network_id = "99999" +section_root = "homepage" +section_segment = 0 + +[[creative_opportunities.slot]] +id = "ad-header-0" +div_id = "ad-header-0" +gam_unit_path = "/{network_id}/example/{section}" +page_patterns = ["/", "/deals", "/deals/*", "/news", "/news/*"] +formats = [{ width = 728, height = 90 }] +``` + +Each section contributes **two** patterns. `*` crosses `/` in this glob +dialect, so `/news/*` matches `/news/a/b` but not the bare `/news` landing +page; emitting only the star form would drop the landing page from the slot. + +Sizes are unioned across pages, so a format that renders only on articles +survives alongside the homepage's. + +### When it keeps literal paths, and when it refuses + +A wrong ad-unit template makes the publisher bid against inventory that does not +exist, so the command prefers a narrow literal path over a plausible guess. + +| Situation | Result | +| --------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| Only one page was crawled | Literal path. One observation cannot distinguish a literal from a template. | +| The ad unit never varied by section | Literal path. | +| A section's slug is not derivable from its URL (`/car-research` requesting `.../carresearch`) | Literal path; the round-trip check catches it. | +| No root page was seen, so `section_root` is unknown | Literal path rather than a guessed fallback. | +| Two path segments could both be the section | No template; the ambiguity is reported. | +| The ad unit varies by device, geo, or anything the URL cannot supply | **No `gam_unit_path` at all** for that slot. | +| Crawled pages report different GAM network ids | The run fails; the pages are not one property. | +| More than a quarter of crawled pages return no slots | The run fails. That is the signature of bot protection serving challenge pages, and writing from it would silently narrow the slot set. | + +Every run checks that the config it produced still loads before replacing the +file, and `--dry-run` runs the same check — a clean preview is evidence the +config loads, not just that it parses. + +### Bounding and steering the crawl + +```bash +# Cover more of a large site. +ts audit ad-templates generate https://publisher.example/ --max-sections 20 --max-pages 41 + +# Audit exactly one page, as earlier releases did. +ts audit ad-templates generate https://publisher.example/ --max-pages 1 + +# Set the patterns yourself; this disables pattern inference entirely. +ts audit ad-templates generate https://publisher.example/ \ + --page-pattern '/' --page-pattern '/news' --page-pattern '/news/*' + +# Preview without writing. +ts audit ad-templates generate https://publisher.example/ --dry-run +``` + +Re-running merges into the existing slots: a slot seen again keeps its +hand-tuned fields and gains this run's patterns, and a hand-written +`gam_unit_path` template is preserved. `--replace` discards existing slots +instead, which also discards any template you wrote by hand. + +Behind bot protection, pass a valid clearance cookie. The crawl reuses one +browser session, so clearance earned on the first page carries to the rest: + +```bash +ts audit ad-templates generate https://publisher.example/ --cookie 'datadome=' +``` + +### Checking for a device split + +Publishers often serve a different ad unit per device +(`/network/desktop/news` against `/network/mobile/news`). A desktop-only crawl +cannot see that — it infers a template correct for desktop and silently wrong +for every mobile impression. + +```bash +ts audit ad-templates generate https://publisher.example/ --profiles desktop,mobile +``` + +Each page is loaded once per profile. Where the profiles disagree, the slot is +written with its div and formats but **no** `gam_unit_path`, so the runtime +falls back to the default unit rather than bidding on one that does not exist. + +### Deploy ordering for templated config + +> **A config containing `section_root` or `section_segment` is not +> rollback-safe.** These keys are rejected outright by a Trusted Server binary +> that predates ad-unit templating, and the rejection fails the _entire_ +> configuration load — not just the ad-template section — so every route serves +> an error. This is a full-site outage, not a degraded ad stack. + +When a run reports that it wrote a `{section}` template: + +1. Deploy the template-aware binary **first**. +2. Then `ts config push`. +3. Do **not** roll that binary back while the config is live. + +A run that did not template writes neither key, and leaves the config exactly as +rollback-safe as it was. + ### Audit safety defaults Every `ts audit` browser session validates TLS certificates. This matters From 90bc61e426c9db7ac9f73de55911926a50ad04c0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 12:20:27 +0530 Subject: [PATCH 263/395] Report why a crawled page yielded no ad slots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live run against a bot-protected site refused with "no slots discovered" and nothing else, because the per-page reasons were collected and then thrown away: `fold_collected` discarded each page's collector warnings, and both refusal paths returned before any note was printed. The guards exist for runs that went wrong, so that is exactly when the reasons matter. Notes are now drained as soon as the crawl finishes, ahead of the refusals, and each page's warnings are attributed to its path. Also name the failure that has no warning of its own. Bot protection commonly answers with 200 and a challenge document rather than a 4xx, so the status check passes, the page settles cleanly, and it simply appears to run no ad stack — indistinguishable from a publisher who genuinely has none, though the operator's next move differs completely. A page carrying almost no scripts and no recognised integrations is now called out as a probable challenge, with the advice to supply a current cookie. Verified against a live protected origin: the run previously reported only that no slots were found; it now identifies the interstitial and says what to do. --- .../src/commands/audit/generate/mod.rs | 74 +++++++++++++++++-- 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index d31030ec7..d74a22e71 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -549,7 +549,7 @@ pub(crate) fn run_update_slots( let root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); let mut table = evidence::EvidenceTable::default(); let mut notes = Vec::new(); - fold_collected(&mut table, &root_url, &root)?; + fold_collected(&mut table, &root_url, &root, &mut notes)?; // One page per section is enough: ad slots repeat per section, so the crawl // is sized by the publisher's taxonomy rather than its catalogue. @@ -563,7 +563,7 @@ pub(crate) fn run_update_slots( if index > 0 { let repeat = first_collector.collect_page(&root_url, request.cookies); match repeat { - Ok(page) => fold_collected(&mut table, &root_url, &page)?, + Ok(page) => fold_collected(&mut table, &root_url, &page, &mut notes)?, Err(error) => notes.push(format!("skipped `{root_url}` on {label}: {error}")), } } @@ -581,8 +581,17 @@ pub(crate) fn run_update_slots( )); } + // Emit what the crawl learned before any refusal below can return early. + // The guards exist precisely for runs that went wrong, so that is when the + // per-page reasons matter most. + emit_notes(out, &mut notes)?; + if table.is_empty() { - return cli_error("no ad-template slots were discovered on any crawled page"); + return cli_error(format!( + "no ad-template slots were discovered on any of the {} crawled page(s); \ + see the notes above for what each page reported", + table.pages().len() + )); } guard_challenge_rate(&table)?; @@ -624,10 +633,7 @@ pub(crate) fn run_update_slots( // preview looked fine" would not be evidence that the config loads. notes.extend(validate::check_candidate(&updated, &existing)?); - for note in ¬es { - writeln!(out, "note: {note}") - .map_err(|error| report_error(format!("failed to write command output: {error}")))?; - } + emit_notes(out, &mut notes)?; if policy.is_some() { writeln!( out, @@ -661,13 +667,65 @@ pub(crate) fn run_update_slots( .map_err(|error| report_error(format!("failed to write command output: {error}"))) } +/// A page carrying fewer scripts than this is not a real publisher page. +/// +/// A production page runs dozens: the ad stack, analytics, consent, and the +/// site's own bundles. A bot-protection interstitial runs its own challenge +/// script and little else. +const INTERSTITIAL_SCRIPT_CEILING: usize = 3; + +/// Whether a page that loaded successfully is nonetheless not the real page. +/// +/// Bot protection commonly answers with **200** and a challenge document rather +/// than a 4xx, so status-code checks pass and the page simply appears to have no +/// ad stack. Left unexplained, that is indistinguishable from a publisher who +/// genuinely runs no ads on that page — and the operator's next move is entirely +/// different in each case. +fn looks_like_an_interstitial(artifact: &AuditArtifact) -> Option { + if artifact.js_asset_count > INTERSTITIAL_SCRIPT_CEILING + || !artifact.detected_integrations.is_empty() + { + return None; + } + Some(format!( + "the page returned successfully but carried only {} script(s) and no recognised \ + integrations, which is the shape of a bot-protection challenge rather than the \ + real page. Supply a current --cookie for the origin", + artifact.js_asset_count + )) +} + +/// Writes and clears the pending notes, so each is reported exactly once. +fn emit_notes(out: &mut dyn Write, notes: &mut Vec) -> CliResult<()> { + for note in notes.drain(..) { + writeln!(out, "note: {note}") + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + Ok(()) +} + /// Discovers a collected page's slots and folds them into `table`. +/// +/// Per-page collector warnings are appended to `notes`. They carry the reason a +/// page came back without slots — a non-2xx main document, a navigation that +/// never settled — which is the difference between "this publisher has no ad +/// stack here" and "bot protection served a challenge". Dropping them leaves +/// the operator with a refusal and no way to act on it. fn fold_collected( table: &mut evidence::EvidenceTable, url: &Url, collected: &collector::CollectedPage, + notes: &mut Vec, ) -> CliResult<()> { + // `analyze_collected_page` already carries the collector's warnings forward, + // so this is the complete set, not a second copy. let artifact = analyze_collected_page(collected)?; + for warning in &artifact.warnings { + notes.push(format!("`{}`: {warning}", url.path())); + } + if let Some(reason) = looks_like_an_interstitial(&artifact) { + notes.push(format!("`{}`: {reason}", url.path())); + } let page_has_prebid = artifact .detected_integrations .iter() @@ -709,7 +767,7 @@ fn crawl_sections( match collected { Ok(page) => { let final_url = page.final_url().unwrap_or_else(|_| url.clone()); - if let Err(error) = fold_collected(table, &final_url, &page) { + if let Err(error) = fold_collected(table, &final_url, &page, notes) { fold_error = Some(error); return Ok(collector::ControlFlow::Stop); } From b8a5e5ca450ff2e7fe550dd2c60b773c22eb4dcc Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 12:24:42 +0530 Subject: [PATCH 264/395] Pace the ad-template crawl and allow a headful browser A live crawl of a bot-protected origin returned the real page for the first request and a challenge for the remaining thirteen. A dead cookie fails on the first page, so that shape points at the session being flagged during the run rather than at the credential. Two contributors, both worth correcting regardless of that diagnosis. The crawl issued its navigations back to back. That is discourteous to the origin on its own terms, and request pacing is among the signals bot protection scores, so an unpaced crawl invites the challenge that empties the rest of the run. `--page-delay-ms` now spaces them, defaulting to 750ms. Headless Chrome is trivially detectable, so an origin that serves the real page to a normal browser may answer the same request headless with a challenge. `--headful` runs a visible browser for the cases where that is the difference. Note that `BrowserConfig` defaults to the *old* headless mode, so simply not requesting new-headless yields a more detectable browser rather than a headful one. Both branches are explicit for that reason. --- .../audit/generate/browser_collector.rs | 73 +++++++++++++++++-- .../src/commands/audit/mod.rs | 18 +++++ 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index eead2d49a..b4ef2badd 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -114,6 +114,10 @@ impl DeviceProfile { #[derive(Debug, Clone, Copy, Default)] pub(crate) struct BrowserAuditCollector { profile: Option, + /// Pause between page loads during a crawl. + page_delay: Duration, + /// Run a visible browser instead of a headless one. + headful: bool, } impl BrowserAuditCollector { @@ -122,6 +126,48 @@ impl BrowserAuditCollector { pub(crate) fn with_profile(profile: DeviceProfile) -> Self { Self { profile: Some(profile), + ..Self::default() + } + } + + /// Sets the pause between page loads. + /// + /// A crawl issues a dozen navigations in a row. Firing them back to back is + /// both discourteous to the origin and self-defeating: request pacing is one + /// of the signals bot protection scores, so an unpaced crawl invites the + /// challenge that empties the rest of the run. + #[must_use] + pub(crate) fn with_page_delay(mut self, delay: Duration) -> Self { + self.page_delay = delay; + self + } + + /// Runs a visible browser rather than a headless one. + /// + /// Headless Chrome is trivially detectable and is scored heavily by bot + /// protection, so an origin that serves a real page to a normal browser may + /// answer the same request headless with a challenge. + #[must_use] + pub(crate) fn headful(mut self, headful: bool) -> Self { + self.headful = headful; + self + } +} + +/// The browser-session knobs one crawl runs under. +#[derive(Debug, Clone, Copy)] +struct SessionSettings { + profile: Option, + page_delay: Duration, + headful: bool, +} + +impl BrowserAuditCollector { + fn session(self) -> SessionSettings { + SessionSettings { + profile: self.profile, + page_delay: self.page_delay, + headful: self.headful, } } } @@ -141,13 +187,13 @@ impl AuditCollector for BrowserAuditCollector { )) })?; - let profile = self.profile; + let settings = self.session(); runtime.block_on(async { let mut collected = None; with_browser( std::slice::from_ref(target_url), cookies, - profile, + settings, &mut |_, result| { collected = Some(result); Ok(ControlFlow::Stop) @@ -176,7 +222,7 @@ impl AuditCollector for BrowserAuditCollector { )) })?; - runtime.block_on(with_browser(targets, cookies, self.profile, on_page)) + runtime.block_on(with_browser(targets, cookies, self.session(), on_page)) } } @@ -191,9 +237,14 @@ impl AuditCollector for BrowserAuditCollector { async fn with_browser( targets: &[Url], cookies: &[(String, String)], - profile: Option, + settings: SessionSettings, sink: PageSink<'_>, ) -> CliResult<()> { + let SessionSettings { + profile, + page_delay, + headful, + } = settings; let chrome_executable = find_browser_executable()?; let user_data_dir = TempDir::new().map_err(|error| { report_error(format!( @@ -207,8 +258,15 @@ async fn with_browser( let mut builder = BrowserConfig::builder() .chrome_executable(chrome_executable) .user_data_dir(user_data_dir.path()) - .new_headless_mode() .respect_https_errors(); + // `BrowserConfig` defaults to the *old* headless mode, which is both more + // detectable and less faithful than either alternative — so both branches + // must be explicit. Omitting the call is not the same as running headful. + builder = if headful { + builder.with_head() + } else { + builder.new_headless_mode() + }; if let Some(profile) = profile { let viewport = profile.viewport(); builder = builder @@ -243,6 +301,11 @@ async fn with_browser( // Sitemap discovery is a whole-site fact, so only the first target pays for it. let mut result = Ok(()); for (index, target) in targets.iter().enumerate() { + // Pace the crawl. Back-to-back navigations are both discourteous to the + // origin and a signal bot protection scores against the session. + if index > 0 && !page_delay.is_zero() { + sleep(page_delay).await; + } let collected = collect_page_from_browser(&mut browser, target, cookies, index == 0).await; match sink(target, collected) { Ok(ControlFlow::Continue) => {} diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index b526a4088..fb3472c28 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -156,6 +156,22 @@ pub(crate) struct AuditAdTemplatesGenerateArgs { /// for any slot where the profiles disagree. #[arg(long, value_delimiter = ',', default_value = "desktop")] pub profiles: Vec, + /// Pause in milliseconds between page loads during the crawl. + /// + /// A crawl issues a dozen navigations in a row. Firing them back to back is + /// discourteous to the origin, and request pacing is one of the signals bot + /// protection scores, so an unpaced crawl can trigger the challenge that + /// empties the rest of the run. + #[arg(long, default_value_t = 750)] + pub page_delay_ms: u64, + /// Run a visible browser instead of a headless one. + /// + /// Headless Chrome is trivially detectable and scored heavily by bot + /// protection, so an origin that serves the real page to a normal browser + /// may answer the same request headless with a challenge. Requires a desktop + /// session; it opens a real window. + #[arg(long)] + pub headful: bool, } impl AuditAdTemplatesGenerateArgs { @@ -240,6 +256,8 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { .iter() .map(|profile| { generate::browser_collector::BrowserAuditCollector::with_profile(*profile) + .with_page_delay(std::time::Duration::from_millis(gen_args.page_delay_ms)) + .headful(gen_args.headful) }) .collect(); let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles From 142e384de5f475e315ecd89eba51a2381b755ae3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 15:23:34 +0530 Subject: [PATCH 265/395] Answer consent APIs and report GPT state during ad-template generate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live audit of a consent-gated publisher reported no ad slots and gave no way to tell why. Two additions, found by debugging exactly that. Publishers gate slot definition behind their consent platform, and a fresh audit profile has no consent cookie, so the crawl never reaches `googletag.defineSlot` and the page looks like it has no ad stack at all. The audit browser now answers the two IAB interfaces every compliant platform exposes, TCF v2 and US Privacy, installed before any page script runs so the real platform finds them already defined. `gdprApplies: false` avoids fabricating a consent string and matches the signal genuinely out-of-scope traffic carries. `--no-assume-consent` observes the un-consented page instead. When the slot registry comes back empty, the run now reports what GPT actually looked like — whether the library reached `apiReady`, how many queued commands never drained, whether `pubads()` exists, and how many scripts the page ran. An empty registry has several very different causes, and the operator's next move differs for each. Against a local proxy this immediately distinguished "GPT never finished loading" from "this page has no ads", which no amount of re-running could have shown before. --- .../audit/generate/browser_collector.rs | 148 +++++++++++++++++- .../src/commands/audit/mod.rs | 10 ++ 2 files changed, 157 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index b4ef2badd..beb5da93d 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -118,6 +118,8 @@ pub(crate) struct BrowserAuditCollector { page_delay: Duration, /// Run a visible browser instead of a headless one. headful: bool, + /// Answer the consent APIs as a consenting reader. + assume_consent: bool, } impl BrowserAuditCollector { @@ -152,6 +154,15 @@ impl BrowserAuditCollector { self.headful = headful; self } + + /// Answers the IAB consent APIs so a gated ad stack initialises. + /// + /// See [`CONSENT_STUB_SCRIPT`] for what is answered and why. + #[must_use] + pub(crate) fn assume_consent(mut self, assume_consent: bool) -> Self { + self.assume_consent = assume_consent; + self + } } /// The browser-session knobs one crawl runs under. @@ -160,6 +171,7 @@ struct SessionSettings { profile: Option, page_delay: Duration, headful: bool, + assume_consent: bool, } impl BrowserAuditCollector { @@ -168,10 +180,97 @@ impl BrowserAuditCollector { profile: self.profile, page_delay: self.page_delay, headful: self.headful, + assume_consent: self.assume_consent, } } } +/// Answers the consent APIs as a consenting, non-GDPR reader. +/// +/// Publishers gate slot definition behind their consent platform, so a browser +/// with no consent cookie never reaches `googletag.defineSlot` and the audit +/// sees a page with no ad stack. That is indistinguishable from a page that +/// genuinely has none, and it is the state every fresh audit profile starts in. +/// +/// Rather than special-casing each vendor, this answers the two IAB interfaces +/// every compliant platform exposes — TCF v2 (`__tcfapi`) and US Privacy +/// (`__uspapi`) — installed before any page script runs so the real platform +/// finds them already defined. `gdprApplies: false` is used deliberately: it +/// needs no fabricated consent string, and it is the same signal the ad stack +/// receives for genuinely out-of-scope traffic. +/// +/// This makes the audit behave like a consenting reader; it does not alter what +/// the publisher's own readers experience. +const CONSENT_STUB_SCRIPT: &str = r#"(() => { + const tcData = { + tcString: '', + tcfPolicyVersion: 2, + cmpId: 0, + cmpVersion: 1, + gdprApplies: false, + eventStatus: 'tcloaded', + cmpStatus: 'loaded', + listenerId: 1, + isServiceSpecific: true, + useNonStandardTexts: false, + purposeOneTreatment: false, + publisherCC: 'US', + purpose: { consents: {}, legitimateInterests: {} }, + vendor: { consents: {}, legitimateInterests: {} }, + specialFeatureOptins: {}, + }; + for (let index = 1; index <= 10; index += 1) { + tcData.purpose.consents[index] = true; + tcData.purpose.legitimateInterests[index] = true; + } + + const tcfapi = (command, version, callback, parameter) => { + if (typeof callback !== 'function') return; + switch (command) { + case 'ping': + callback({ + gdprApplies: false, + cmpLoaded: true, + cmpStatus: 'loaded', + displayStatus: 'hidden', + apiVersion: '2.0', + cmpId: 0, + }, true); + break; + case 'addEventListener': + case 'getTCData': + callback(tcData, true); + break; + case 'removeEventListener': + callback(true, true); + break; + default: + callback(tcData, true); + } + }; + + const uspapi = (command, version, callback) => { + if (typeof callback !== 'function') return; + callback({ version: 1, uspString: '1---' }, true); + }; + + // Non-writable so the real platform cannot replace these and re-gate the + // page; a failed assignment is the intended outcome. + const pin = (name, value) => { + try { + Object.defineProperty(window, name, { + value, + writable: false, + configurable: false, + }); + } catch (error) { + /* already pinned */ + } + }; + pin('__tcfapi', tcfapi); + pin('__uspapi', uspapi); +})();"#; + impl AuditCollector for BrowserAuditCollector { fn collect_page( &self, @@ -244,6 +343,7 @@ async fn with_browser( profile, page_delay, headful, + assume_consent, } = settings; let chrome_executable = find_browser_executable()?; let user_data_dir = TempDir::new().map_err(|error| { @@ -306,7 +406,9 @@ async fn with_browser( if index > 0 && !page_delay.is_zero() { sleep(page_delay).await; } - let collected = collect_page_from_browser(&mut browser, target, cookies, index == 0).await; + let collected = + collect_page_from_browser(&mut browser, target, cookies, index == 0, assume_consent) + .await; match sink(target, collected) { Ok(ControlFlow::Continue) => {} Ok(ControlFlow::Stop) => break, @@ -346,11 +448,22 @@ async fn collect_page_from_browser( target_url: &Url, cookies: &[(String, String)], discover_sitemap: bool, + assume_consent: bool, ) -> CliResult { let page = browser.new_page("about:blank").await.map_err(|error| { report_error(format!("failed to create browser page for audit: {error}")) })?; + // Must run before any page script, so the consent platform finds the APIs + // already answered rather than installing its own gate. + if assume_consent { + page.evaluate_on_new_document(CONSENT_STUB_SCRIPT) + .await + .map_err(|error| { + report_error(format!("failed to install the consent stub: {error}")) + })?; + } + // Set operator-supplied cookies before navigating so the origin sees an // authenticated session on the first request. Scoping each to the target URL // lets Chrome infer domain/path. @@ -470,6 +583,19 @@ async fn collect_page_from_browser( // page keeps its link graph in the framework payload, so parsing the raw // HTML finds only a fraction of the site's sections. Best-effort — an empty // list just means crawl planning falls back to other sources. + // When the registry is empty, report what GPT actually looked like. An + // empty registry has several very different causes — the library never + // loaded, it loaded but the command queue never drained, or slots really + // are absent — and the operator's next move differs for each. + if gpt_slots.is_empty() + && let Ok(result) = page.evaluate(GPT_DIAGNOSTIC_SCRIPT).await + && let Ok(state) = result.into_value::() + { + warnings.push(format!( + "no GPT slots in the registry; googletag state: {state}" + )); + } + let links: Vec = match page.evaluate(LINKS_SCRIPT).await { Ok(result) => result.into_value().unwrap_or_default(), Err(_) => Vec::new(), @@ -626,6 +752,26 @@ const SITEMAP_SCRIPT: &str = r#"async () => { return pages.slice(0, 5000); }"#; +/// Reports the observable state of GPT, for pages whose registry came back empty. +const GPT_DIAGNOSTIC_SCRIPT: &str = r#"() => { + const tag = window.googletag; + const count = (() => { + try { return tag.pubads().getSlots().length } catch (error) { return -1 } + })(); + return { + googletag: typeof tag, + api_ready: !!(tag && tag.apiReady), + cmd_pending: tag && tag.cmd && typeof tag.cmd.length === 'number' ? tag.cmd.length : -1, + has_pubads: !!(tag && typeof tag.pubads === 'function'), + slots: count, + tcfapi: typeof window.__tcfapi, + scripts: document.scripts.length, + ts_ad_slots: (() => { + try { return (window.tsjs && window.tsjs.adSlots || []).length } catch (error) { return -1 } + })(), + }; +}"#; + /// Reads the live GPT slot registry into `{gam_unit_path, div_id, sizes}` rows. /// /// Mirrors the ad-template verifier's `getSlots()` scrape: it defends against a diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index fb3472c28..777b09a2e 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -172,6 +172,15 @@ pub(crate) struct AuditAdTemplatesGenerateArgs { /// session; it opens a real window. #[arg(long)] pub headful: bool, + /// Do not answer the IAB consent APIs on behalf of the audit browser. + /// + /// Publishers gate slot definition behind their consent platform, and a + /// fresh audit profile has no consent cookie, so by default the crawl + /// answers the standard TCF v2 and US Privacy interfaces as a consenting, + /// out-of-scope reader. Without that, such a site reports no ad slots at + /// all. Pass this to observe the un-consented page instead. + #[arg(long)] + pub no_assume_consent: bool, } impl AuditAdTemplatesGenerateArgs { @@ -258,6 +267,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { generate::browser_collector::BrowserAuditCollector::with_profile(*profile) .with_page_delay(std::time::Duration::from_millis(gen_args.page_delay_ms)) .headful(gen_args.headful) + .assume_consent(!gen_args.no_assume_consent) }) .collect(); let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles From f5c861620c0daf3fefd74fd422a88a5eddb72221 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 15:55:59 +0530 Subject: [PATCH 266/395] Audit through a proxy and collapse lowercase React div-id tokens Verified against a live publisher served by `ts dev proxy`, which surfaced two defects that no fixture could. `normalize_div_stem` matched only the uppercase React `_R_` marker. React also emits the lowercase `_r_0_` form client-side, and the token changes on every render, so a slot arrived as `ad-header-0-_r_0_` on one page and `ad-header-0-_r_8_` on the next. One logical slot fragmented into a new key per page: the written `div_id` would never match at runtime, and template inference saw no slot twice, so it had no variation to reason about and kept every path literal. Collapsing the lowercase form is what lets the crawl rediscover `/{network_id}/autoblog/{section}` from live evidence. Add `--browser-proxy` so the audit can run against a production hostname served locally, which keeps the page's origin, cookie scope, and any origin checks in the ad stack matching production rather than `localhost`. `--danger-accept- invalid-certs` covers a MITM certificate whose CA the throwaway browser profile does not trust. Note that chromiumoxide builds each Chrome flag by prefixing `--` to the arg key, so a pre-formatted `--flag=value` string becomes `----flag=value` and is silently dropped. Both the new proxy flags and the existing mobile user-agent override were written that way; the user-agent override had therefore never taken effect. Both now pass `(key, value)` pairs. --- .../audit/generate/browser_collector.rs | 65 +++++++++++++++++-- .../src/commands/audit/generate/gpt_slots.rs | 52 ++++++++++++++- .../src/commands/audit/mod.rs | 19 ++++++ 3 files changed, 127 insertions(+), 9 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index beb5da93d..148bbf2aa 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -111,7 +111,7 @@ impl DeviceProfile { } /// Collects pages through a local Chrome, emulating one device profile. -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Default)] pub(crate) struct BrowserAuditCollector { profile: Option, /// Pause between page loads during a crawl. @@ -120,6 +120,10 @@ pub(crate) struct BrowserAuditCollector { headful: bool, /// Answer the consent APIs as a consenting reader. assume_consent: bool, + /// Route the browser through this proxy, as `host:port`. + proxy: Option, + /// Accept TLS certificates that do not validate. + accept_invalid_certs: bool, } impl BrowserAuditCollector { @@ -163,24 +167,50 @@ impl BrowserAuditCollector { self.assume_consent = assume_consent; self } + + /// Routes the browser through `proxy` (`host:port`). + /// + /// Lets the audit run against a production hostname served by a local + /// MITM proxy, so the page's origin, cookie scope, and any origin checks in + /// the ad stack match production rather than `localhost`. + #[must_use] + pub(crate) fn with_proxy(mut self, proxy: Option) -> Self { + self.proxy = proxy; + self + } + + /// Accepts TLS certificates that do not validate. + /// + /// Needed when a MITM proxy presents a certificate from a CA the browser + /// profile does not trust. Dangerous against a real origin: see the flag + /// documentation. + #[must_use] + pub(crate) fn accept_invalid_certs(mut self, accept: bool) -> Self { + self.accept_invalid_certs = accept; + self + } } /// The browser-session knobs one crawl runs under. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] struct SessionSettings { profile: Option, page_delay: Duration, headful: bool, assume_consent: bool, + proxy: Option, + accept_invalid_certs: bool, } impl BrowserAuditCollector { - fn session(self) -> SessionSettings { + fn session(&self) -> SessionSettings { SessionSettings { profile: self.profile, page_delay: self.page_delay, headful: self.headful, assume_consent: self.assume_consent, + proxy: self.proxy.clone(), + accept_invalid_certs: self.accept_invalid_certs, } } } @@ -344,6 +374,8 @@ async fn with_browser( page_delay, headful, assume_consent, + proxy, + accept_invalid_certs, } = settings; let chrome_executable = find_browser_executable()?; let user_data_dir = TempDir::new().map_err(|error| { @@ -357,8 +389,26 @@ async fn with_browser( // the config with slots of its choosing. Validate certificates. let mut builder = BrowserConfig::builder() .chrome_executable(chrome_executable) - .user_data_dir(user_data_dir.path()) - .respect_https_errors(); + .user_data_dir(user_data_dir.path()); + if !accept_invalid_certs { + builder = builder.respect_https_errors(); + } + if let Some(proxy) = &proxy { + // Chrome ignores a scheme-less `--proxy-server` value, silently sending + // traffic direct instead, so normalise it. `<-loopback>` keeps Chrome + // from bypassing the proxy for loopback hosts, which is exactly the case + // a local MITM proxy serves. + let endpoint = if proxy.contains("://") { + proxy.clone() + } else { + format!("http://{proxy}") + }; + // Keys carry no `--`: chromiumoxide adds it, so a pre-formatted + // `--flag=value` string becomes `----flag=value` and is ignored. + builder = builder + .arg(("proxy-server", endpoint.as_str())) + .arg(("proxy-bypass-list", "<-loopback>")); + } // `BrowserConfig` defaults to the *old* headless mode, which is both more // detectable and less faithful than either alternative — so both branches // must be explicit. Omitting the call is not the same as running headful. @@ -375,7 +425,10 @@ async fn with_browser( if let Some(user_agent) = profile.user_agent() { // Ad stacks branch on the user agent as well as the viewport, so // emulating size alone can still return desktop ad units. - builder = builder.arg(format!("--user-agent={user_agent}")); + // Key without the `--`: chromiumoxide prefixes it, so passing a + // pre-formatted `--flag=value` string yields `----flag=value`, + // which Chrome silently ignores. + builder = builder.arg(("user-agent", user_agent)); } } let config = builder.build().map_err(|error| { diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index 365a5b696..d7b0b6bfe 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -33,6 +33,21 @@ use crate::commands::audit::generate::collector::{CollectedGptSlot, CollectedReq static HEX_HASH_SEGMENT: LazyLock = LazyLock::new(|| Regex::new(r"-[0-9a-f]{16,}(?:-|$)").expect("should compile hex hash regex")); +/// Matches a React `useId` token, which changes on every render. +/// +/// React emits these in both cases — `_R_3f_` from a server render and `_r_0_` +/// from a client one — so matching only the uppercase form leaves the lowercase +/// variant in the stem. That is not merely untidy: the suffix differs per +/// render, so one logical slot fragments into a new key on every page, which +/// both breaks runtime div matching and starves template inference of the +/// repeated observations it needs. +/// +/// The uppercase form is distinctive enough to match bare. The lowercase one is +/// anchored (`_r_`, a short alphanumeric run, `_`) so an ordinary id that merely +/// contains `_r_` keeps its full stem. +static REACT_USE_ID: LazyLock = + LazyLock::new(|| Regex::new(r"_R_|_r_[0-9a-z]{1,8}_").expect("should compile react id regex")); + /// Hosts that serve GPT `gampad/ads` requests. const GAMPAD_HOSTS: &[&str] = &["securepubads.g.doubleclick.net", "pubads.g.doubleclick.net"]; @@ -185,12 +200,13 @@ fn is_usable_unit_path(path: &str) -> bool { /// a valid **prefix** of the live div id, which is how verify matches slots. /// /// `div-gpt-ad-leaderboard-1` (stable) is unchanged; `ad-header-0-_R_9sl…-container` -/// → `ad-header-0`; `ad-in_content-de66…f272-in_content-0` → `ad-in_content`. +/// and `ad-header-0-_r_8_` → `ad-header-0`; `ad-in_content-de66…f272-in_content-0` +/// → `ad-in_content`. fn normalize_div_stem(div_id: &str) -> String { let stem = div_id.strip_suffix("-container").unwrap_or(div_id); let mut cut = stem.len(); - if let Some(pos) = stem.find("_R_") { - cut = cut.min(pos); + if let Some(matched) = REACT_USE_ID.find(stem) { + cut = cut.min(matched.start()); } if let Some(matched) = HEX_HASH_SEGMENT.find(stem) { cut = cut.min(matched.start()); @@ -496,6 +512,36 @@ mod tests { } } + #[test] + fn lowercase_react_use_id_suffixes_collapse_to_one_slot() { + // React emits `_r_0_` client-side and `_R_3f_` server-side, and the + // token changes per render. Leaving it in the stem fragments one slot + // into a new key on every page, which starves template inference. + for volatile in [ + "ad-header-0-_r_0_", + "ad-header-0-_r_8_", + "ad-header-0-_r_a_", + "ad-header-0-_R_3f_", + ] { + let registry = vec![registry_slot("/123/site/news", volatile, &[(728, 90)])]; + let discovered = discover_gpt_slots(®istry, &[], false); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "`{volatile}` should normalize to a stable stem" + ); + } + } + + #[test] + fn an_ordinary_id_containing_r_is_left_alone() { + // The React shape is anchored, so a legitimate id keeps its full stem. + let registry = vec![registry_slot("/123/site/news", "ad_r_rail", &[(300, 250)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].div_id, "ad_r_rail"); + } + #[test] fn registry_slot_with_brace_in_unit_path_is_skipped() { // `gam_unit_path` is a template and there is no escape syntax, so a diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 777b09a2e..d5787665e 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -181,6 +181,23 @@ pub(crate) struct AuditAdTemplatesGenerateArgs { /// all. Pass this to observe the un-consented page instead. #[arg(long)] pub no_assume_consent: bool, + /// Route the audit browser through a proxy, as `host:port`. + /// + /// Pairs with `ts dev proxy`, which serves a production hostname from a + /// local Trusted Server. Auditing through it means the page's origin, + /// cookie scope, and any origin checks in the ad stack match production + /// rather than `localhost`. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Accept TLS certificates that do not validate. + /// + /// DANGEROUS against a real origin: the audit sends any `--cookie` session + /// upstream and treats the response as evidence, so an invalid certificate + /// could mean an impersonator is harvesting the session and fabricating the + /// result. Intended for a local MITM proxy whose CA the browser profile does + /// not trust; prefer installing that CA (`ts dev proxy ca`) over this flag. + #[arg(long)] + pub danger_accept_invalid_certs: bool, } impl AuditAdTemplatesGenerateArgs { @@ -268,6 +285,8 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { .with_page_delay(std::time::Duration::from_millis(gen_args.page_delay_ms)) .headful(gen_args.headful) .assume_consent(!gen_args.no_assume_consent) + .with_proxy(gen_args.browser_proxy.clone()) + .accept_invalid_certs(gen_args.danger_accept_invalid_certs) }) .collect(); let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles From 230958b6aaeb51880cb7da4b53a4d3db0e2706e2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 16:09:59 +0530 Subject: [PATCH 267/395] Refuse ad-template slots that are one placement under per-render div ids A live crawl produced fourteen slots where four were real. Ten were two placements repeated: an ad stack built its div ids from a per-render token, so the same placement arrived under a new key on every page. Written verbatim those ids match nothing at runtime, and the fragmentation also starves template inference, which needs to observe a slot more than once. Detect it from evidence rather than by pattern-matching token shapes, since each stack invents its own and the previous two forms already needed separate handling. Candidates share an identical ad-unit path and identical formats; what separates a fragmented placement from two legitimate siblings on one unit is co-occurrence. Real siblings appear together on a page, while fragments never do, because each page yields exactly one of them. Fragments are reported and skipped rather than written. The report names the observed ids and the stable prefix they share, so the operator can add the placement once with a prefix they know survives a render. That prefix is deliberately not written as a `div_id`: it reaches only as far as the observed tokens happen to agree, so it would match this crawl's ids and miss the next render's. Verified live: the run that previously wrote fourteen slots now writes the four real ones and explains the two it declined. --- .../src/commands/audit/generate/evidence.rs | 200 ++++++++++++++++++ .../src/commands/audit/generate/mod.rs | 35 ++- 2 files changed, 234 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs index bfa004b94..b2404da54 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -61,6 +61,62 @@ impl SlotEvidence { } } +/// Slots grouped by the shape that would make them one placement: an identical +/// ad-unit path and an identical format set. +type SlotsByShape<'a> = BTreeMap<(String, Vec<(u32, u32)>), Vec<&'a SlotEvidence>>; + +/// Several observed slots that are really one placement under volatile div ids. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct FragmentGroup { + /// The volatile div ids observed, in evidence order. + pub(super) div_ids: Vec, + /// The ad-unit path every fragment shared. + pub(super) unit_path: String, + /// The stable prefix the ids share, when they share a useful one. + /// + /// Offered to the operator as a starting point only. It is deliberately not + /// written as a `div_id`: the shared prefix reaches only as far as the + /// *observed* tokens happen to agree, so it would keep matching this crawl's + /// ids and stop matching the next render's. + pub(super) suggested_prefix: Option, +} + +/// Whether no two slots were ever seen on the same page. +fn pages_are_disjoint(slots: &[&SlotEvidence]) -> bool { + for (index, slot) in slots.iter().enumerate() { + let pages = slot.paths(); + if slots[index + 1..] + .iter() + .any(|other| other.paths().intersection(&pages).next().is_some()) + { + return false; + } + } + true +} + +/// The longest prefix the div ids share, trimmed back to a separator. +/// +/// Trimming matters: the raw common prefix usually ends mid-token (the leading +/// digits of a timestamp two fragments happen to share), which is worse than +/// useless as a suggestion. Cutting at the last `-` or `_` yields the part a +/// human would recognise as the placement's name. +fn shared_div_prefix(slots: &[&SlotEvidence]) -> Option { + let mut prefix: &str = slots.first()?.div_id.as_str(); + for slot in &slots[1..] { + let shared = slot + .div_id + .char_indices() + .zip(prefix.chars()) + .take_while(|((_, left), right)| left == right) + .count(); + prefix = &prefix[..shared]; + } + let trimmed = prefix.trim_end_matches(|ch: char| ch != '-' && ch != '_'); + let candidate = trimmed.trim_end_matches(['-', '_']); + (!candidate.is_empty()).then(|| candidate.to_string()) +} + /// Slot evidence accumulated across every collected page. #[derive(Debug, Clone, Default)] pub(super) struct EvidenceTable { @@ -144,6 +200,46 @@ impl EvidenceTable { self.slots.is_empty() } + /// Groups of slots that are one slot wearing a different div id per page. + /// + /// Some ad stacks build div ids from a per-render token — a timestamp, a + /// framework id — so the same placement arrives under a new key on every + /// page. Written verbatim those ids never match at runtime, and the + /// fragmentation also starves template inference, which needs to see one + /// slot more than once. + /// + /// Detection is by evidence rather than by guessing at token shapes, because + /// each stack invents its own. Candidates share an identical ad-unit path and + /// identical formats; what separates a fragmented slot from two legitimate + /// siblings on the same unit is **co-occurrence**. Real siblings appear + /// together on a page; fragments of one slot never do, because each page + /// produces exactly one of them. + pub(super) fn fragmented_slots(&self) -> Vec { + let mut by_shape: SlotsByShape<'_> = BTreeMap::new(); + for slot in self.slots() { + // Only slots pinned to exactly one unit path can be compared this + // way; a slot whose unit varies is inference's problem, not this one. + let units = slot.unit_paths(); + if units.len() != 1 { + continue; + } + let unit = (*units.iter().next().expect("one unit path")).to_string(); + let formats: Vec<(u32, u32)> = slot.formats.iter().copied().collect(); + by_shape.entry((unit, formats)).or_default().push(slot); + } + + by_shape + .into_iter() + .filter(|(_, slots)| slots.len() > 1) + .filter(|(_, slots)| pages_are_disjoint(slots)) + .map(|((unit_path, _), slots)| FragmentGroup { + div_ids: slots.iter().map(|slot| slot.div_id.clone()).collect(), + unit_path, + suggested_prefix: shared_div_prefix(&slots), + }) + .collect() + } + /// The single GAM network id observed across the crawl. /// /// # Errors @@ -291,6 +387,110 @@ mod tests { ); } + #[test] + fn one_placement_under_per_render_div_ids_is_detected() { + // The live shape: a timestamped token means each page yields a new key + // for the same placement. Same unit, same formats, never co-occurring. + let mut table = EvidenceTable::default(); + for (path, div) in [ + ( + "/features/a", + "rh-gam-kso_26329268ce6Bj0uc8sL0_ei_overlay_1", + ), + ("/news/b", "rh-gam-kso_26329269aoYmv4RQyN3n_ei_overlay_1"), + ("/deals/c", "rh-gam-kso_26329270mYPDB3tz8cpB_ei_overlay_1"), + ] { + table.fold_page( + path, + &page(&[("/99/site_Overlay", div, &[(300, 250)])], false), + ); + } + + let groups = table.fragmented_slots(); + + assert_eq!(groups.len(), 1, "the three fragments should form one group"); + assert_eq!(groups[0].div_ids.len(), 3); + assert_eq!(groups[0].unit_path, "/99/site_Overlay"); + assert_eq!( + groups[0].suggested_prefix.as_deref(), + Some("rh-gam-kso"), + "the suggestion should be trimmed back off the volatile token" + ); + } + + #[test] + fn genuine_siblings_on_one_unit_are_not_treated_as_fragments() { + // Two real in-content positions can share a unit path and formats. What + // distinguishes them from fragments is that they appear *together* on a + // page, so refusing to write them would lose real inventory. + let mut table = EvidenceTable::default(); + table.fold_page( + "/news/story", + &page( + &[ + ("/99/site/news", "ad-in_content-1", &[(300, 250)]), + ("/99/site/news", "ad-in_content-2", &[(300, 250)]), + ], + false, + ), + ); + + assert!( + table.fragmented_slots().is_empty(), + "co-occurring slots are siblings, not fragments" + ); + } + + #[test] + fn slots_differing_in_formats_are_not_fragments() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "slot-aaaa", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "slot-bbbb", &[(728, 90)])], false), + ); + + assert!( + table.fragmented_slots().is_empty(), + "a differing format set means these are different placements" + ); + } + + #[test] + fn a_slot_seen_alone_is_never_a_fragment() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "only-slot", &[(300, 250)])], false), + ); + + assert!(table.fragmented_slots().is_empty()); + } + + #[test] + fn fragments_with_no_shared_prefix_report_none() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "alpha-1111", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "beta-2222", &[(300, 250)])], false), + ); + + let groups = table.fragmented_slots(); + + assert_eq!(groups.len(), 1); + assert_eq!( + groups[0].suggested_prefix, None, + "unrelated ids should not produce a misleading suggestion" + ); + } + #[test] fn conflicting_network_ids_are_a_hard_error() { let mut table = EvidenceTable::default(); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index d74a22e71..2a2d4b11b 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -614,7 +614,32 @@ pub(crate) fn run_update_slots( .as_ref() .and_then(|outcome| outcome.policy.clone()); - let slots = build_render_slots(&table, inference.as_ref(), policy.as_ref(), request)?; + // Slots that are one placement wearing a per-render div id cannot be + // written: the ids never match at runtime. Report them so the operator can + // add the placement once with a prefix they know is stable. + let fragmented = table.fragmented_slots(); + for group in &fragmented { + let suggestion = group.suggested_prefix.as_deref().map_or_else( + || "no stable prefix was shared".to_string(), + |prefix| format!("they share the prefix `{prefix}`"), + ); + notes.push(format!( + "skipped {} slot(s) that look like one placement under a per-render div id on \ + `{}` ({}); {suggestion}. Add it once by hand with a div_id prefix that is \ + stable across renders", + group.div_ids.len(), + group.unit_path, + group.div_ids.join(", "), + )); + } + + let slots = build_render_slots( + &table, + inference.as_ref(), + policy.as_ref(), + request, + &fragmented, + )?; let merged = slot_toml::merge_render_slots(request.existing_creative, slots, request.replace); let rendered_slots = render_slots(&merged); let updated = splice_creative_slots( @@ -804,7 +829,12 @@ fn build_render_slots( inference: Option<&unit_template::InferenceOutcome>, policy: Option<&unit_template::SectionPolicy>, request: &UpdateSlotsRequest<'_>, + fragmented: &[evidence::FragmentGroup], ) -> CliResult> { + let skip: std::collections::BTreeSet<&str> = fragmented + .iter() + .flat_map(|group| group.div_ids.iter().map(String::as_str)) + .collect(); // Explicit `--page-pattern` values are an operator override: they apply to // every slot and disable inference from observed paths entirely. let explicit = !request.page_patterns.is_empty(); @@ -815,6 +845,9 @@ fn build_render_slots( let mut slots = Vec::with_capacity(table.slot_count()); for slot in table.slots() { + if skip.contains(slot.div_id.as_str()) { + continue; + } let patterns = if explicit { request.page_patterns.to_vec() } else { From 64b941478197fddd15d3e8eee636c24394931cf8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 16:39:06 +0530 Subject: [PATCH 268/395] Document the ad-template crawl options added after the first draft MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four options landed after the command was first documented and were never written up: request pacing, a headful browser, the consent answer, and auditing through a local proxy. Each exists because a live audit of a protected publisher failed without it, so the reason belongs alongside the flag. Consent gets its own section because the failure is silent. A publisher gates slot definition behind its consent platform, the audit runs in a throwaway profile with no consent cookie, and the result is a page that appears to have no ad stack at all — indistinguishable from one that genuinely has none. Also record that an empty slot registry now reports GPT's observable state, which is what separates "the library never loaded" from "this page has no ads". Proxy auditing gets a section because `ts dev proxy` is how a production hostname is served locally, and matching the production origin matters for cookie scope and for origin checks inside the ad stack. Note the caveat that a local Trusted Server injects its own configured slots, so a run through the proxy can rediscover config it already has. Finally, describe how per-render div ids are detected and reported, including why the suggested prefix is offered but never written. --- docs/guide/cli.md | 76 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/docs/guide/cli.md b/docs/guide/cli.md index b0aeb612b..9c31fb56d 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -229,12 +229,84 @@ hand-tuned fields and gains this run's patterns, and a hand-written instead, which also discards any template you wrote by hand. Behind bot protection, pass a valid clearance cookie. The crawl reuses one -browser session, so clearance earned on the first page carries to the rest: +browser session, so clearance earned on the first page carries to the rest, and +`--page-delay-ms` spaces the requests — an unpaced crawl is both discourteous to +the origin and likelier to be challenged partway through: ```bash -ts audit ad-templates generate https://publisher.example/ --cookie 'datadome=' +ts audit ad-templates generate https://publisher.example/ \ + --cookie '=' --page-delay-ms 1500 +``` + +Some origins refuse a headless browser outright regardless of the cookie. +`--headful` runs a visible one, which is also the quickest way to _see_ whether +a challenge is being shown: + +```bash +ts audit ad-templates generate https://publisher.example/ --headful +``` + +### Sites behind a consent platform + +Publishers gate slot definition behind their consent platform, and the audit +runs in a throwaway browser profile with no consent cookie. Left alone, such a +site defines no slots at all and looks identical to a site with no ad stack. + +The crawl therefore answers the two IAB interfaces every compliant platform +exposes — TCF v2 and US Privacy — as a consenting, out-of-scope reader, before +any page script runs. This changes only what the audit browser sees; it does not +affect the publisher's own readers. Pass `--no-assume-consent` to observe the +un-consented page instead. + +When a page still yields no slots, the run reports GPT's observable state — +whether the library reached `apiReady`, how many queued commands never drained, +how many scripts ran. An empty slot registry has several very different causes, +and that line distinguishes them. + +### Auditing a production hostname served locally + +`ts dev proxy` serves a production hostname from a local Trusted Server. +Auditing through it keeps the page's origin, cookie scope, and any origin checks +in the ad stack matching production rather than `localhost`: + +```bash +ts dev proxy --map www.publisher.example=127.0.0.1:7676 --upstream-plaintext --rewrite-host + +ts audit ad-templates generate https://www.publisher.example/ \ + --browser-proxy 127.0.0.1:18080 --danger-accept-invalid-certs ``` +`--danger-accept-invalid-certs` covers the proxy's MITM certificate when the +throwaway browser profile does not trust its CA; installing that CA +(`ts dev proxy ca`) is preferable. Against a real origin the flag is dangerous — +the audit sends any `--cookie` session upstream and treats the response as +evidence, so an invalid certificate could mean an impersonator is both +harvesting the session and fabricating the result. + +Note that a local Trusted Server injects its own configured slots into the page, +so a run through the proxy can rediscover config it already has. Slot ids that +are absent from the current config are the publisher's own. + +### Slots that change div id on every render + +Some ad stacks build div ids from a per-render token, so one placement arrives +under a new id on every page. Those ids match nothing at runtime, so the run +declines to write them and reports the group instead: + +```text +note: skipped 3 slot(s) that look like one placement under a per-render div id + on `/12345678/example.com_Overlay` (kso_2632930aBc_overlay_1, …); + they share the prefix `kso`. Add it once by hand with a div_id prefix + that is stable across renders +``` + +The detection is by evidence, not by recognising token shapes: candidates share +an ad-unit path and formats, and what separates a fragmented placement from two +legitimate siblings on one unit is co-occurrence — real siblings appear together +on a page, fragments never do. The suggested prefix is a starting point only, not +written as a `div_id`, because it reaches only as far as the observed tokens +happen to agree. + ### Checking for a device split Publishers often serve a different ad unit per device From 7d2ba2db1214086bb6e0fe1cf9a6ca2f61cff3e0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 21:15:37 +0530 Subject: [PATCH 269/395] Revise SSAT debug comment sensitivity model --- ...-07-20-ssat-debug-comment-config-design.md | 113 +++++++++++------- 1 file changed, 71 insertions(+), 42 deletions(-) diff --git a/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md b/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md index d8b33d3c4..1d7de7177 100644 --- a/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md +++ b/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md @@ -2,7 +2,7 @@ **Date:** 2026-07-20 -**Status:** Proposed +**Status:** Approved; security model revised 2026-08-17 **Issue:** [IABTechLab/trusted-server#935](https://github.com/IABTechLab/trusted-server/issues/935) — "For SSAT, make debug comment configurable" @@ -22,21 +22,23 @@ alongside the existing bool, with: 1. Section toggles (provider responses / mediator response / bids array). 2. A configurable subset of an expanded, still-hardcoded metadata allowlist. -3. A `verbosity` switch (`redacted` default, `full` opt-in) that bypasses the - allowlist and creative truncation entirely for deep debugging. +3. A three-level `verbosity` switch: `redacted` (safe default), `upstream` + (bounded provider error text), and `full` (raw metadata and creatives). ## Goals 1. Let an operator omit sections of the dump to keep it small/focused. 2. Let an operator select which of the already-safe metadata keys to surface. -3. Surface `http_status` and `upstream_message` — already captured - server-side, currently absent from the allowlist by omission, not - design — so "was it a 400, and for what reason" is answerable in the - default (redacted) mode. +3. Surface `http_status` in the safe default mode, and make the already-captured + `upstream_message` available through an explicit intermediate `upstream` + mode so "was it a 400, and for what reason" is answerable without enabling + the complete provider dump. Upstream text is provider-controlled and may + echo sensitive request values, so it is not part of `redacted` mode. 4. Provide an explicit, loudly-documented `full` mode for the rare case where an operator needs the raw per-bidder request/response (PBS `debug.httpcalls`) to diagnose a specific auction, accepting the PII exposure that implies. -5. Never let configuration weaken the fail-closed guarantee in redacted mode: +5. Never let section or metadata-key configuration weaken the fail-closed + guarantee in redacted mode: identity-bearing data (device IP, geo, `user.ext.eids`, TC consent string) must be unreachable via `metadata_keys` regardless of what an operator configures. @@ -48,11 +50,11 @@ alongside the existing bool, with: - Configurable size limits. Explicitly declined — `MAX_BID_CREATIVE_DUMP_BYTES` (512) and `MAX_AUCTION_DEBUG_DUMP_BYTES` (256KB) stay hardcoded constants; neither becomes operator-tunable. Note this does **not** mean both apply in - both verbosity modes: `MAX_AUCTION_DEBUG_DUMP_BYTES` (the 256KB total cap) - is unconditional in both `Redacted` and `Full`, but `MAX_BID_CREATIVE_DUMP_BYTES` - (the 512-byte per-bid preview) is `Redacted`-only by design — `Full` skips - creative truncation entirely (see Rendering / Data Flow). "Hardcoded" means - "not config-driven," not "applied unconditionally in every mode." + every verbosity mode: `MAX_AUCTION_DEBUG_DUMP_BYTES` (the 256KB total cap) + is unconditional, while `MAX_BID_CREATIVE_DUMP_BYTES` (the 512-byte per-bid + preview) applies to `Redacted` and `Upstream`; `Full` skips creative + truncation entirely (see Rendering / Data Flow). "Hardcoded" means "not + config-driven," not "applied unconditionally in every mode." - Adding failure-reason instrumentation to provider adapters that don't capture any today. Notably, `AuctionResponse::no_bid()` ([auction/types.rs:296](../../../crates/trusted-server-core/src/auction/types.rs#L296)) @@ -70,7 +72,7 @@ alongside the existing bool, with: (client JS state vs. HTML comment), no interaction with this design. - Tightening `Bid`-level fields (`Bid.metadata`, `nurl`, `burl`) to a fail-closed allowlist. These already pass through `redact_bid_for_dump` - unfiltered today in both verbosity modes + unfiltered today in all verbosity modes ([publisher.rs:966-972](../../../crates/trusted-server-core/src/publisher.rs#L966-L972)), pre-existing and tracked separately as issue #925. Unaffected by, and orthogonal to, this design. @@ -99,6 +101,9 @@ server-side by the prebid integration: - `upstream_message` / `upstream_message_truncated` — the actual PBS error body text, only populated when `[integrations.prebid] debug = true` ([prebid.rs:2038-2046](../../../crates/trusted-server-core/src/integrations/prebid.rs#L2038-L2046)). + The text is bounded but provider-controlled, and may echo identifiers or + other request values. It is therefore available only in `upstream` and + `full` modes, never in the safe `redacted` default. Also captured server-side, but deliberately excluded from the allowlist and staying that way in redacted mode: the raw `debug` subtree @@ -150,14 +155,18 @@ pub struct AuctionDebugCommentOptions { /// `Redacted` (default): `metadata_keys` subset only, creative preview /// truncated to `MAX_BID_CREATIVE_DUMP_BYTES`. + /// `Upstream`: the redacted fields plus bounded provider-controlled + /// `upstream_message` and `upstream_message_truncated`; creatives remain + /// truncated and all other metadata remains filtered. /// `Full`: raw `response.metadata` verbatim, including the `debug` /// subtree (httpcalls/resolvedrequest — device IP, geo, eids, TC consent /// string — when `integrations.prebid.debug` is also on), and no /// creative truncation. The 256KB total dump cap and comment-terminator /// neutralization still apply. /// - /// NEVER enable `Full` in production: identity-bearing request/response - /// data becomes visible to any visitor via view-source. + /// NEVER enable `Upstream` or `Full` in production: provider-controlled + /// text or identity-bearing request/response data becomes visible to any + /// visitor via view-source. #[serde(default)] pub verbosity: AuctionDebugCommentVerbosity, } @@ -190,6 +199,7 @@ impl AuctionDebugCommentOptions { pub enum AuctionDebugCommentVerbosity { #[default] Redacted, + Upstream, Full, } ``` @@ -206,19 +216,18 @@ serde's per-field `#[serde(default = "...")]` must agree. `AUCTION_DEBUG_METADATA_ALLOWLIST` moves from `publisher.rs` into `settings.rs` as the single canonical superset (publisher.rs imports it; it's the same list used both as `default_auction_debug_metadata_keys()`'s return value and as the -fail-closed intersection filter — see Security Invariants). New expanded list +fail-closed intersection filter — see Security Invariants). The safe list drops the old allowlist's `"status"` key (verified: no production code path writes `response.metadata["status"]` — only an unrelated `telemetry.rs` test does — so this is an intentional cleanup of a key nothing ever populates, not -an accidental narrowing): +an accidental narrowing). Provider-controlled upstream text is deliberately +excluded and handled by the verbosity branch instead: ```rust const AUCTION_DEBUG_METADATA_ALLOWLIST: &[&str] = &[ "error_type", "http_status", "message", - "upstream_message", - "upstream_message_truncated", "responsetimemillis", "errors", "warnings", @@ -268,14 +277,26 @@ if options.include_mediator_response `options.metadata_keys ∩ AUCTION_DEBUG_METADATA_ALLOWLIST`. The intersection is computed here, at the render call — this is the actual security boundary, not the config struct itself. +- `Upstream`: the same filtered metadata as `Redacted`, plus + `upstream_message` and `upstream_message_truncated` when present. These keys + are selected by the enum branch, never by `metadata_keys`, so configuration + cannot widen the boundary accidentally. This mode is explicitly sensitive + because upstream text may echo request data. - `Full`: `metadata` = `response.metadata.clone()`, unfiltered. - `bids` = `[]` when `!options.include_bids`; otherwise each bid goes through `redact_bid_for_dump(bid, options)`. +The implementation keeps the two upstream keys in a separate hardcoded const, +`AUCTION_DEBUG_UPSTREAM_METADATA_KEYS`. It first builds the safe configured +intersection, then adds those keys only for `Upstream`; `Full` remains a direct +copy of all metadata. Pattern-based text redaction is deliberately avoided: +arbitrary identifiers and consent values cannot be recognized exhaustively, +so presenting such filtering as safe would weaken the fail-closed contract. + `redact_bid_for_dump(bid, options)`: -- `Redacted`: `creative` truncated to `MAX_BID_CREATIVE_DUMP_BYTES` (512), - as today. +- `Redacted` and `Upstream`: `creative` truncated to + `MAX_BID_CREATIVE_DUMP_BYTES` (512), as today. - `Full`: `creative` passed through untruncated. Unconditional regardless of `options` (safety nets, not redaction controls): @@ -309,16 +330,20 @@ if settings.debug.auction_html_comment { intersection silently drops it. This must hold even when the operator's intent is clearly to widen access; fail-closed means the config cannot widen the boundary, only narrow what's already inside it. -2. **`Full` verbosity is the only path to identity-bearing data**, and it - requires two independent, explicit opt-ins to have any effect for prebid: - `debug.auction_html_comment_options.verbosity = "full"` AND - `integrations.prebid.debug = true` (the latter is what makes PBS return the - `debug.httpcalls` subtree at all). Neither flag alone exposes anything new. -3. **Comment-terminator neutralization and the total byte cap are +2. **Provider-controlled text requires an explicit sensitivity mode.** + `upstream_message` is never in the redacted allowlist. It appears only when + verbosity is `upstream` or `full`, and only when + `integrations.prebid.debug = true` captured it. Operators must treat + `upstream` as potentially sensitive because a provider can echo identifiers + or request values in an error message. +3. **Raw structured identity data requires `Full`.** The PBS `debug.httpcalls` + and `resolvedrequest` subtrees remain excluded in `upstream` mode and require + `verbosity = "full"` plus `integrations.prebid.debug = true`. +4. **Comment-terminator neutralization and the total byte cap are unconditional** — they are HTML-injection and page-bloat safety nets, not privacy controls, and must never be gated behind `verbosity` or any other option. -4. **Bad `verbosity` values fail config load**, not silently fall back to +5. **Bad `verbosity` values fail config load**, not silently fall back to `Redacted`. An unrecognized string is a serde deserialize error at startup — loud failure over silent (mis)interpretation. @@ -326,6 +351,9 @@ if settings.debug.auction_html_comment { - **`metadata_keys = []`**: valid; yields `metadata: {}` per response. An operator can explicitly request zero metadata while still seeing bids/status. +- **`verbosity=Upstream` is intentionally narrower than `Full`**: it adds only + the bounded upstream error message fields. It does not expose arbitrary + metadata, raw PBS requests/responses, or untruncated creatives. - **`verbosity=Full` value is provider-dependent**: only `prebid.rs` populates `metadata["debug"]` today. For `aps` and other direct integrations, `Full` only removes creative truncation and the metadata filter — there's no @@ -338,12 +366,9 @@ if settings.debug.auction_html_comment { rather than exceptional. No new per-provider cap is being added (see Non-goals) — this is a known, accepted tradeoff. - **Default output changes for existing operators**: anyone already running - `auction_html_comment = true` gets 3 additional keys - (`http_status`, `upstream_message`, `upstream_message_truncated`) in the - default dump with zero config changes on their part. Not a security - regression — still fail-closed, still no identity data — but it is a - default-output change worth calling out explicitly rather than smuggling in - silently. + `auction_html_comment = true` gets the structured `http_status` key in the + default dump with zero config changes. Provider-controlled upstream text + requires the explicit `upstream` or `full` mode. ## Testing Strategy @@ -352,9 +377,13 @@ Arrange-Act-Assert, matching existing `auction_debug_comment_*` tests — no `rstest` in this file today): - `default_options_reproduce_current_behavior` — regression: default struct - vs. today's hardcoded output, identical except: the unused `status` key - (never written by any production path) is gone, and the 3 new keys - (`http_status`, `upstream_message`, `upstream_message_truncated`) are added. + vs. today's hardcoded output, identical except the unused metadata `status` + key is gone and structured `http_status` is added. The fixture must contain + the relevant metadata so the assertion proves the behavior. +- `configured_metadata_subset_only_includes_selected_safe_keys` +- `redacted_mode_never_surfaces_provider_controlled_upstream_message` — use an + identity-shaped marker to prove the default privacy boundary. +- `upstream_mode_includes_bounded_upstream_message_but_not_debug_subtree` - `metadata_keys_empty_yields_empty_metadata_object` - `metadata_keys_attack_vector_debug_key_never_surfaces_in_redacted_mode` — configuring `metadata_keys = ["debug"]` under `Redacted` still produces no @@ -386,12 +415,12 @@ include_mediator_response = true include_bids = true metadata_keys = [ "error_type", "http_status", "message", - "upstream_message", "upstream_message_truncated", "responsetimemillis", "errors", "warnings", "bidstatus", ] -# "redacted" (default) or "full". NEVER "full" in production — exposes -# device IP, geo, consent string, and eids via view-source when -# integrations.prebid.debug is also enabled. +# "redacted" (default), "upstream", or "full". +# "upstream" exposes provider-controlled error text, which may echo request +# data. "full" additionally exposes raw metadata and untruncated creatives. +# Never use either sensitive mode in production. verbosity = "redacted" ``` From 3715f274bafdc22f933a348c08571b0452cbf92b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 21:20:31 +0530 Subject: [PATCH 270/395] Tighten SSAT debug metadata privacy model --- ...-07-20-ssat-debug-comment-config-design.md | 79 +++++++++++-------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md b/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md index 1d7de7177..18d7bea29 100644 --- a/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md +++ b/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md @@ -2,7 +2,7 @@ **Date:** 2026-07-20 -**Status:** Approved; security model revised 2026-08-17 +**Status:** Proposed security revision; awaiting approval **Issue:** [IABTechLab/trusted-server#935](https://github.com/IABTechLab/trusted-server/issues/935) — "For SSAT, make debug comment configurable" @@ -21,16 +21,18 @@ This design adds a config table, `[debug.auction_html_comment_options]`, alongside the existing bool, with: 1. Section toggles (provider responses / mediator response / bids array). -2. A configurable subset of an expanded, still-hardcoded metadata allowlist. +2. A configurable subset of a revised, still-hardcoded safe metadata + allowlist. 3. A three-level `verbosity` switch: `redacted` (safe default), `upstream` - (bounded provider error text), and `full` (raw metadata and creatives). + (provider diagnostic text), and `full` (raw metadata and creatives). ## Goals 1. Let an operator omit sections of the dump to keep it small/focused. 2. Let an operator select which of the already-safe metadata keys to surface. 3. Surface `http_status` in the safe default mode, and make the already-captured - `upstream_message` available through an explicit intermediate `upstream` + provider-controlled diagnostic text available through an explicit + intermediate `upstream` mode so "was it a 400, and for what reason" is answerable without enabling the complete provider dump. Upstream text is provider-controlled and may echo sensitive request values, so it is not part of `redacted` mode. @@ -105,6 +107,10 @@ server-side by the prebid integration: other request values. It is therefore available only in `upstream` and `full` modes, never in the safe `redacted` default. +The current allowlist's `errors` and `warnings` values also come verbatim from +PBS. They carry the same echo risk as `upstream_message`, so this design moves +them out of `redacted` and makes them available only in `upstream` and `full`. + Also captured server-side, but deliberately excluded from the allowlist and staying that way in redacted mode: the raw `debug` subtree (`httpcalls`/`resolvedrequest`) that PBS returns when `integrations.prebid.debug` @@ -124,8 +130,9 @@ pub struct DebugConfig { // inject_adm_for_testing)... /// Behavior of the ts-debug comment. Only consulted when - /// `auction_html_comment` is true. Defaults reproduce today's fixed - /// output plus the two allowlist additions below. + /// `auction_html_comment` is true. Defaults preserve today's enabled + /// sections and redacted creative previews while using the revised safe + /// metadata allowlist documented below. #[serde(default)] pub auction_html_comment_options: AuctionDebugCommentOptions, } @@ -155,8 +162,8 @@ pub struct AuctionDebugCommentOptions { /// `Redacted` (default): `metadata_keys` subset only, creative preview /// truncated to `MAX_BID_CREATIVE_DUMP_BYTES`. - /// `Upstream`: the redacted fields plus bounded provider-controlled - /// `upstream_message` and `upstream_message_truncated`; creatives remain + /// `Upstream`: the redacted fields plus provider-controlled `errors`, + /// `warnings`, and bounded `upstream_message` fields; creatives remain /// truncated and all other metadata remains filtered. /// `Full`: raw `response.metadata` verbatim, including the `debug` /// subtree (httpcalls/resolvedrequest — device IP, geo, eids, TC consent @@ -229,8 +236,6 @@ const AUCTION_DEBUG_METADATA_ALLOWLIST: &[&str] = &[ "http_status", "message", "responsetimemillis", - "errors", - "warnings", "bidstatus", ]; ``` @@ -277,16 +282,18 @@ if options.include_mediator_response `options.metadata_keys ∩ AUCTION_DEBUG_METADATA_ALLOWLIST`. The intersection is computed here, at the render call — this is the actual security boundary, not the config struct itself. -- `Upstream`: the same filtered metadata as `Redacted`, plus - `upstream_message` and `upstream_message_truncated` when present. These keys - are selected by the enum branch, never by `metadata_keys`, so configuration - cannot widen the boundary accidentally. This mode is explicitly sensitive - because upstream text may echo request data. +- `Upstream`: the same filtered metadata as `Redacted`, plus `errors`, + `warnings`, `upstream_message`, and `upstream_message_truncated` when + present. These keys are selected by the enum branch, never by + `metadata_keys`, so configuration cannot widen the boundary accidentally. + This mode is explicitly sensitive because PBS supplies these values and its + diagnostic text may echo request data. - `Full`: `metadata` = `response.metadata.clone()`, unfiltered. - `bids` = `[]` when `!options.include_bids`; otherwise each bid goes through `redact_bid_for_dump(bid, options)`. -The implementation keeps the two upstream keys in a separate hardcoded const, +The implementation keeps the four provider-controlled diagnostic keys in a +separate hardcoded const, `AUCTION_DEBUG_UPSTREAM_METADATA_KEYS`. It first builds the safe configured intersection, then adds those keys only for `Upstream`; `Full` remains a direct copy of all metadata. Pattern-based text redaction is deliberately avoided: @@ -295,8 +302,10 @@ so presenting such filtering as safe would weaken the fail-closed contract. `redact_bid_for_dump(bid, options)`: -- `Redacted` and `Upstream`: `creative` truncated to - `MAX_BID_CREATIVE_DUMP_BYTES` (512), as today. +- `Redacted` and `Upstream`: retain at most + `MAX_BID_CREATIVE_DUMP_BYTES` (512) bytes of the creative, as today. The + appended truncation marker is outside that retained-prefix limit, so the + rendered field may be slightly larger than 512 bytes. - `Full`: `creative` passed through untruncated. Unconditional regardless of `options` (safety nets, not redaction controls): @@ -331,11 +340,12 @@ if settings.debug.auction_html_comment { intent is clearly to widen access; fail-closed means the config cannot widen the boundary, only narrow what's already inside it. 2. **Provider-controlled text requires an explicit sensitivity mode.** - `upstream_message` is never in the redacted allowlist. It appears only when - verbosity is `upstream` or `full`, and only when - `integrations.prebid.debug = true` captured it. Operators must treat - `upstream` as potentially sensitive because a provider can echo identifiers - or request values in an error message. + `errors`, `warnings`, `upstream_message`, and + `upstream_message_truncated` are never in the redacted allowlist. They + appear only when verbosity is `upstream` or `full`; the upstream-message + fields additionally require `integrations.prebid.debug = true` to capture + them. Operators must treat `upstream` as potentially sensitive because a + provider can echo identifiers or request values in diagnostic text. 3. **Raw structured identity data requires `Full`.** The PBS `debug.httpcalls` and `resolvedrequest` subtrees remain excluded in `upstream` mode and require `verbosity = "full"` plus `integrations.prebid.debug = true`. @@ -349,11 +359,15 @@ if settings.debug.auction_html_comment { ## Edge Cases and Behavior Changes -- **`metadata_keys = []`**: valid; yields `metadata: {}` per response. An - operator can explicitly request zero metadata while still seeing bids/status. +- **`metadata_keys = []`**: valid; yields no configured safe metadata in + `Redacted`. In `Upstream`, provider-controlled diagnostic fields may still + be added by that explicit sensitivity mode. `Full` ignores the selector and + copies all metadata. An operator can therefore request zero metadata only + while remaining in `Redacted`. - **`verbosity=Upstream` is intentionally narrower than `Full`**: it adds only - the bounded upstream error message fields. It does not expose arbitrary - metadata, raw PBS requests/responses, or untruncated creatives. + the provider-controlled `errors`/`warnings` diagnostics and bounded upstream + error message fields. It does not expose arbitrary metadata, raw PBS + requests/responses, or untruncated creatives. - **`verbosity=Full` value is provider-dependent**: only `prebid.rs` populates `metadata["debug"]` today. For `aps` and other direct integrations, `Full` only removes creative truncation and the metadata filter — there's no @@ -381,10 +395,11 @@ Arrange-Act-Assert, matching existing `auction_debug_comment_*` tests — no key is gone and structured `http_status` is added. The fixture must contain the relevant metadata so the assertion proves the behavior. - `configured_metadata_subset_only_includes_selected_safe_keys` -- `redacted_mode_never_surfaces_provider_controlled_upstream_message` — use an - identity-shaped marker to prove the default privacy boundary. -- `upstream_mode_includes_bounded_upstream_message_but_not_debug_subtree` -- `metadata_keys_empty_yields_empty_metadata_object` +- `redacted_mode_never_surfaces_provider_controlled_diagnostic_text` — put + identity-shaped markers in `errors`, `warnings`, and `upstream_message` to + prove the default privacy boundary. +- `upstream_mode_includes_provider_diagnostics_but_not_debug_subtree` +- `metadata_keys_empty_yields_empty_safe_metadata_in_redacted` - `metadata_keys_attack_vector_debug_key_never_surfaces_in_redacted_mode` — configuring `metadata_keys = ["debug"]` under `Redacted` still produces no `debug` key. This is the load-bearing security test for this whole design. @@ -415,7 +430,7 @@ include_mediator_response = true include_bids = true metadata_keys = [ "error_type", "http_status", "message", - "responsetimemillis", "errors", "warnings", "bidstatus", + "responsetimemillis", "bidstatus", ] # "redacted" (default), "upstream", or "full". # "upstream" exposes provider-controlled error text, which may echo request From 8a61578820139378bb9576527455fa1063f72ad8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 21:25:00 +0530 Subject: [PATCH 271/395] Validate redacted SSAT metadata by schema --- ...-07-20-ssat-debug-comment-config-design.md | 139 +++++++++++------- 1 file changed, 88 insertions(+), 51 deletions(-) diff --git a/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md b/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md index 18d7bea29..a9038d7c1 100644 --- a/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md +++ b/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md @@ -23,16 +23,18 @@ alongside the existing bool, with: 1. Section toggles (provider responses / mediator response / bids array). 2. A configurable subset of a revised, still-hardcoded safe metadata allowlist. -3. A three-level `verbosity` switch: `redacted` (safe default), `upstream` - (provider diagnostic text), and `full` (raw metadata and creatives). +3. A three-level `verbosity` switch: `redacted` (validated response metadata by + default), `upstream` (provider diagnostic text), and `full` (raw metadata + and creatives). ## Goals 1. Let an operator omit sections of the dump to keep it small/focused. -2. Let an operator select which of the already-safe metadata keys to surface. -3. Surface `http_status` in the safe default mode, and make the already-captured - provider-controlled diagnostic text available through an explicit - intermediate `upstream` +2. Let an operator select which validated, server-generated metadata fields to + surface. +3. Surface `http_status` in the validated default response metadata, and make + the already-captured provider-controlled diagnostic text available through + an explicit intermediate `upstream` mode so "was it a 400, and for what reason" is answerable without enabling the complete provider dump. Upstream text is provider-controlled and may echo sensitive request values, so it is not part of `redacted` mode. @@ -40,10 +42,9 @@ alongside the existing bool, with: an operator needs the raw per-bidder request/response (PBS `debug.httpcalls`) to diagnose a specific auction, accepting the PII exposure that implies. 5. Never let section or metadata-key configuration weaken the fail-closed - guarantee in redacted mode: - identity-bearing data (device IP, geo, `user.ext.eids`, TC consent string) - must be unreachable via `metadata_keys` regardless of what an operator - configures. + response-metadata guarantee in redacted mode: identity-bearing response + metadata (device IP, geo, `user.ext.eids`, TC consent string) must be + unreachable via `metadata_keys` regardless of what an operator configures. ## Non-goals @@ -107,9 +108,11 @@ server-side by the prebid integration: other request values. It is therefore available only in `upstream` and `full` modes, never in the safe `redacted` default. -The current allowlist's `errors` and `warnings` values also come verbatim from -PBS. They carry the same echo risk as `upstream_message`, so this design moves -them out of `redacted` and makes them available only in `upstream` and `full`. +The current allowlist's `errors`, `warnings`, `responsetimemillis`, and +`bidstatus` values also come verbatim from untyped PBS JSON. Even fields that +normally contain numbers or status labels could instead contain arbitrary +strings or nested identity data. This design therefore moves all four out of +`redacted` and makes them available only in `upstream` and `full`. Also captured server-side, but deliberately excluded from the allowlist and staying that way in redacted mode: the raw `debug` subtree @@ -162,9 +165,10 @@ pub struct AuctionDebugCommentOptions { /// `Redacted` (default): `metadata_keys` subset only, creative preview /// truncated to `MAX_BID_CREATIVE_DUMP_BYTES`. - /// `Upstream`: the redacted fields plus provider-controlled `errors`, - /// `warnings`, and bounded `upstream_message` fields; creatives remain - /// truncated and all other metadata remains filtered. + /// `Upstream`: the redacted fields plus provider-controlled diagnostics + /// (`errors`, `warnings`, response timings, bid statuses, and bounded + /// upstream-message fields); creatives remain truncated and all other + /// metadata remains filtered. /// `Full`: raw `response.metadata` verbatim, including the `debug` /// subtree (httpcalls/resolvedrequest — device IP, geo, eids, TC consent /// string — when `integrations.prebid.debug` is also on), and no @@ -235,11 +239,29 @@ const AUCTION_DEBUG_METADATA_ALLOWLIST: &[&str] = &[ "error_type", "http_status", "message", - "responsetimemillis", - "bidstatus", ]; ``` +The allowlist controls selectable field names, but name matching alone is not +the security boundary. Rendering validates and normalizes each selected value: + +- `error_type` is copied only when it is one of the server's known static + classifications (`parse_response`, `launch_failed`, `transport`, `timeout`, + or `http_status`). Unknown strings and non-string values are dropped. +- `http_status` is copied only when it is an integer in `100..=599`. +- `message` is never copied from `response.metadata`. When selected, it is + generated by the renderer from the validated classification and optional + validated HTTP status using fixed server-owned wording: + `parse_response` → `Provider response could not be parsed`, `launch_failed` + → `Provider launch failed`, `transport` → `Provider request failed`, + `timeout` → `Provider request timed out`, and `http_status` → + `Provider returned HTTP {status}` (or `Provider returned an HTTP error` when + a valid status is absent). Unknown or invalid classifications produce no + message. + +This makes `redacted` fail closed at the rendering boundary even if a future +provider writes an unexpected value beneath a familiar metadata key. + `Settings::finalize_deserialized` ([settings.rs:2038-2059](../../../crates/trusted-server-core/src/settings.rs#L2038-L2059)) — the associated fn that already calls `settings.integrations.normalize(); @@ -278,21 +300,21 @@ if options.include_mediator_response `redact_response_for_dump(response, options)`: -- `Redacted`: `metadata` = `response.metadata` filtered to - `options.metadata_keys ∩ AUCTION_DEBUG_METADATA_ALLOWLIST`. The intersection - is computed here, at the render call — this is the actual security - boundary, not the config struct itself. -- `Upstream`: the same filtered metadata as `Redacted`, plus `errors`, - `warnings`, `upstream_message`, and `upstream_message_truncated` when - present. These keys are selected by the enum branch, never by - `metadata_keys`, so configuration cannot widen the boundary accidentally. - This mode is explicitly sensitive because PBS supplies these values and its - diagnostic text may echo request data. +- `Redacted`: `metadata` = the validated and normalized safe fields selected by + `options.metadata_keys ∩ AUCTION_DEBUG_METADATA_ALLOWLIST`. Both the + name intersection and the per-field schema checks are computed here, at the + render call — this is the actual security boundary, not the config struct. +- `Upstream`: the same validated metadata as `Redacted`, plus `errors`, + `warnings`, `responsetimemillis`, `bidstatus`, `upstream_message`, and + `upstream_message_truncated` when present. These keys are selected by the + enum branch, never by `metadata_keys`, so configuration cannot widen the + boundary accidentally. This mode is explicitly sensitive because PBS + supplies these untyped values and its diagnostics may echo request data. - `Full`: `metadata` = `response.metadata.clone()`, unfiltered. - `bids` = `[]` when `!options.include_bids`; otherwise each bid goes through `redact_bid_for_dump(bid, options)`. -The implementation keeps the four provider-controlled diagnostic keys in a +The implementation keeps the six provider-controlled diagnostic keys in a separate hardcoded const, `AUCTION_DEBUG_UPSTREAM_METADATA_KEYS`. It first builds the safe configured intersection, then adds those keys only for `Upstream`; `Full` remains a direct @@ -338,14 +360,17 @@ if settings.debug.auction_html_comment { superset) in `metadata_keys` has zero effect in `Redacted` mode — the intersection silently drops it. This must hold even when the operator's intent is clearly to widen access; fail-closed means the config cannot - widen the boundary, only narrow what's already inside it. + widen the boundary, only narrow what's already inside it. A matching key is + still dropped unless its value passes that field's strict schema; `message` + is generated from validated fields rather than copied. 2. **Provider-controlled text requires an explicit sensitivity mode.** - `errors`, `warnings`, `upstream_message`, and - `upstream_message_truncated` are never in the redacted allowlist. They - appear only when verbosity is `upstream` or `full`; the upstream-message - fields additionally require `integrations.prebid.debug = true` to capture - them. Operators must treat `upstream` as potentially sensitive because a - provider can echo identifiers or request values in diagnostic text. + `errors`, `warnings`, `responsetimemillis`, `bidstatus`, + `upstream_message`, and `upstream_message_truncated` are never in the + redacted allowlist. They appear only when verbosity is `upstream` or `full`; + the upstream-message fields additionally require + `integrations.prebid.debug = true` to capture them. Operators must treat + `upstream` as potentially sensitive because untyped provider values can + contain or echo identifiers and request data. 3. **Raw structured identity data requires `Full`.** The PBS `debug.httpcalls` and `resolvedrequest` subtrees remain excluded in `upstream` mode and require `verbosity = "full"` plus `integrations.prebid.debug = true`. @@ -356,6 +381,11 @@ if settings.debug.auction_html_comment { 5. **Bad `verbosity` values fail config load**, not silently fall back to `Redacted`. An unrecognized string is a serde deserialize error at startup — loud failure over silent (mis)interpretation. +6. **The redacted guarantee is limited to response-level metadata.** Existing + bid-level metadata, `nurl`/`burl`, and the retained creative prefix are not + covered by this change; issue #925 tracks tightening those fields. The mode + must be documented as safer response metadata, not as a fully anonymized + auction dump. ## Edge Cases and Behavior Changes @@ -365,9 +395,9 @@ if settings.debug.auction_html_comment { copies all metadata. An operator can therefore request zero metadata only while remaining in `Redacted`. - **`verbosity=Upstream` is intentionally narrower than `Full`**: it adds only - the provider-controlled `errors`/`warnings` diagnostics and bounded upstream - error message fields. It does not expose arbitrary metadata, raw PBS - requests/responses, or untruncated creatives. + the provider-controlled errors, warnings, response timings, bid statuses, + and bounded upstream-message fields. It does not expose other arbitrary + metadata, raw PBS requests/responses, or untruncated creatives. - **`verbosity=Full` value is provider-dependent**: only `prebid.rs` populates `metadata["debug"]` today. For `aps` and other direct integrations, `Full` only removes creative truncation and the metadata filter — there's no @@ -380,9 +410,11 @@ if settings.debug.auction_html_comment { rather than exceptional. No new per-provider cap is being added (see Non-goals) — this is a known, accepted tradeoff. - **Default output changes for existing operators**: anyone already running - `auction_html_comment = true` gets the structured `http_status` key in the - default dump with zero config changes. Provider-controlled upstream text - requires the explicit `upstream` or `full` mode. + `auction_html_comment = true` gets validated `error_type`, `http_status`, and + a renderer-generated `message` in the default dump with zero config changes. + The old untyped `errors`, `warnings`, `responsetimemillis`, and `bidstatus` + fields move behind explicit `upstream` or `full` mode; the unused `status` + key is removed. ## Testing Strategy @@ -390,14 +422,18 @@ All in the existing `publisher.rs` test module (plain `#[test]` fns, Arrange-Act-Assert, matching existing `auction_debug_comment_*` tests — no `rstest` in this file today): -- `default_options_reproduce_current_behavior` — regression: default struct - vs. today's hardcoded output, identical except the unused metadata `status` - key is gone and structured `http_status` is added. The fixture must contain - the relevant metadata so the assertion proves the behavior. +- `default_options_apply_safe_response_metadata_schema` — regression: default + struct keeps today's sections and creative preview while exposing only + validated `error_type`, `http_status`, and renderer-generated `message`. + The fixture must also contain the newly excluded legacy keys so the assertion + proves they do not remain in the default output. - `configured_metadata_subset_only_includes_selected_safe_keys` - `redacted_mode_never_surfaces_provider_controlled_diagnostic_text` — put - identity-shaped markers in `errors`, `warnings`, and `upstream_message` to - prove the default privacy boundary. + identity-shaped markers in every upstream diagnostic key to prove the + response-metadata boundary. +- `redacted_mode_rejects_wrong_types_and_unknown_error_classifications` — put + strings and nested identity-shaped JSON under every safe key; assert only + valid values survive and `message` is regenerated rather than copied. - `upstream_mode_includes_provider_diagnostics_but_not_debug_subtree` - `metadata_keys_empty_yields_empty_safe_metadata_in_redacted` - `metadata_keys_attack_vector_debug_key_never_surfaces_in_redacted_mode` — @@ -420,8 +456,10 @@ Arrange-Act-Assert, matching existing `auction_debug_comment_*` tests — no ```toml [debug] -# NEVER enable in production. Injects a redacted per-provider auction dump -# before . See [debug.auction_html_comment_options] for content control. +# NEVER enable in production. Injects an auction dump before . +# "redacted" validates response metadata but still includes existing bid-level +# fields and creative previews; it is not a fully anonymized dump. +# See [debug.auction_html_comment_options] for content control. auction_html_comment = false [debug.auction_html_comment_options] @@ -430,7 +468,6 @@ include_mediator_response = true include_bids = true metadata_keys = [ "error_type", "http_status", "message", - "responsetimemillis", "bidstatus", ] # "redacted" (default), "upstream", or "full". # "upstream" exposes provider-controlled error text, which may echo request From de3db0a9ed0d52713415d3ff9c82e580631d66a8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 21:27:16 +0530 Subject: [PATCH 272/395] Clarify SSAT upstream safety contract --- .../2026-07-20-ssat-debug-comment-config-design.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md b/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md index a9038d7c1..1c73fd668 100644 --- a/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md +++ b/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md @@ -371,9 +371,12 @@ if settings.debug.auction_html_comment { `integrations.prebid.debug = true` to capture them. Operators must treat `upstream` as potentially sensitive because untyped provider values can contain or echo identifiers and request data. -3. **Raw structured identity data requires `Full`.** The PBS `debug.httpcalls` - and `resolvedrequest` subtrees remain excluded in `upstream` mode and require - `verbosity = "full"` plus `integrations.prebid.debug = true`. +3. **The complete PBS debug subtree requires `Full`.** The PBS + `debug.httpcalls` and `resolvedrequest` subtrees remain excluded in + `upstream` mode and require `verbosity = "full"` plus + `integrations.prebid.debug = true`. This does not make `Upstream` + identity-safe: its six untyped diagnostic values may themselves contain + nested identity or request data, as stated above. 4. **Comment-terminator neutralization and the total byte cap are unconditional** — they are HTML-injection and page-bloat safety nets, not privacy controls, and must never be gated behind `verbosity` or any other @@ -435,6 +438,7 @@ Arrange-Act-Assert, matching existing `auction_debug_comment_*` tests — no strings and nested identity-shaped JSON under every safe key; assert only valid values survive and `message` is regenerated rather than copied. - `upstream_mode_includes_provider_diagnostics_but_not_debug_subtree` +- `verbosity_upstream_still_truncates_creative` - `metadata_keys_empty_yields_empty_safe_metadata_in_redacted` - `metadata_keys_attack_vector_debug_key_never_surfaces_in_redacted_mode` — configuring `metadata_keys = ["debug"]` under `Redacted` still produces no From 1af743c1b2992bd7c736cc736f55bff4e4b066f9 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 21:33:55 +0530 Subject: [PATCH 273/395] Update SSAT debug comment implementation plan --- .../2026-07-20-ssat-debug-comment-config.md | 852 +++++++----------- 1 file changed, 316 insertions(+), 536 deletions(-) diff --git a/docs/superpowers/plans/2026-07-20-ssat-debug-comment-config.md b/docs/superpowers/plans/2026-07-20-ssat-debug-comment-config.md index a1ef9d0c7..e83548659 100644 --- a/docs/superpowers/plans/2026-07-20-ssat-debug-comment-config.md +++ b/docs/superpowers/plans/2026-07-20-ssat-debug-comment-config.md @@ -2,701 +2,481 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Make the SSAT `` auction dump configurable — section toggles, a metadata-key subset, and an opt-in `Full` verbosity that surfaces raw per-bidder request/response data — while keeping the existing fail-closed redaction unconditional. +**Goal:** Finish the SSAT debug-comment configuration with three explicit sensitivity modes while making default response-level metadata fail closed by both key and value schema. -**Architecture:** One new config struct (`AuctionDebugCommentOptions`) and one new enum (`AuctionDebugCommentVerbosity`) in `settings.rs`, threaded as a parameter through the three existing render functions in `publisher.rs`. No new files, no new crates. +**Architecture:** Keep the existing configuration types in `settings.rs` and rendering in `publisher.rs`. Add an `Upstream` enum branch and a separate provider-diagnostic key set. In `Redacted`, reconstruct a small metadata object from validated values instead of copying arbitrary JSON; in `Upstream`, add only six named provider diagnostics; in `Full`, retain the existing raw-metadata behavior. -**Tech Stack:** Rust, serde, existing `trusted-server-core` auction/settings modules. +**Tech Stack:** Rust, serde/serde_json, TOML, existing `trusted-server-core` tests and target-specific Cargo aliases. -**Spec:** `docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md` — read it first for the full rationale (security invariants, non-goals, edge cases). This plan implements it; it doesn't re-derive it. +**Spec:** `docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md` --- ## File Structure -| File | Responsibility | -| --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `crates/trusted-server-core/src/settings.rs` | `AuctionDebugCommentOptions`, `AuctionDebugCommentVerbosity`, `AUCTION_DEBUG_METADATA_ALLOWLIST`, wiring into `DebugConfig` and `finalize_deserialized` | -| `crates/trusted-server-core/src/publisher.rs` | `redact_response_for_dump`, `redact_bid_for_dump`, `prepend_auction_debug_comment` — all three gain an `options` parameter; production + test call sites updated; new tests | -| `trusted-server.example.toml` | Document the new `[debug.auction_html_comment_options]` table | +| File | Responsibility | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | +| `crates/trusted-server-core/src/settings.rs` | Public config schema, safe selector keys, upstream diagnostic keys, normalization, config tests | +| `crates/trusted-server-core/src/publisher.rs` | Schema validation, safe message generation, three rendering branches, section/size protections, renderer tests | +| `trusted-server.example.toml` | Operator-facing mode and privacy documentation | +| `docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md` | Approved security contract; no behavioral edits during implementation unless a discovered contradiction is brought back for approval | -No file split needed — both touched files stay well under the codebase's existing size (settings.rs and publisher.rs are already large multi-struct files; this adds one cohesive struct + enum to each, following the file's existing pattern of many sibling config structs). +No new production file or dependency is needed. The existing large modules already own these responsibilities, so splitting them during this security fix would add unrelated churn. --- -### Task 1: Config struct in settings.rs +### Task 1: Lock down the configuration boundary **Files:** -- Modify: `crates/trusted-server-core/src/settings.rs:1894-1924` (the `DebugConfig` block) -- Modify: `crates/trusted-server-core/src/settings.rs:2038-2059` (`finalize_deserialized`) -- Test: same file, `#[cfg(test)] mod tests` block (search for an existing `mod tests` near the bottom of settings.rs to append into) +- Modify: `crates/trusted-server-core/src/settings.rs:1890-2020` +- Test: `crates/trusted-server-core/src/settings.rs:2700-2770` -- [ ] **Step 1: Write the failing test for `AuctionDebugCommentOptions::default()`** +- [ ] **Step 1: Write failing settings tests** -Add to the settings.rs test module: +Update `auction_debug_comment_options_default_matches_serde_defaults` to require exactly the safe selector keys: ```rust -#[test] -fn auction_debug_comment_options_default_matches_serde_defaults() { - let opts = AuctionDebugCommentOptions::default(); - assert!(opts.include_provider_responses, "should default to true"); - assert!(opts.include_mediator_response, "should default to true"); - assert!(opts.include_bids, "should default to true"); - assert_eq!( - opts.metadata_keys, - AUCTION_DEBUG_METADATA_ALLOWLIST - .iter() - .map(|s| s.to_string()) - .collect::>(), - "should default metadata_keys to the full allowlist" - ); - assert_eq!( - opts.verbosity, - AuctionDebugCommentVerbosity::Redacted, - "should default to Redacted" - ); -} +assert_eq!( + opts.metadata_keys, + vec![ + "error_type".to_string(), + "http_status".to_string(), + "message".to_string(), + ], + "should default to only schema-validated response metadata" +); +``` -#[test] -fn auction_debug_comment_options_normalize_trims_and_drops_empty_keys() { - let mut opts = AuctionDebugCommentOptions { - metadata_keys: vec![" status ".to_string(), "".to_string(), "warnings".to_string()], - ..AuctionDebugCommentOptions::default() - }; - opts.normalize(); - assert_eq!(opts.metadata_keys, vec!["status".to_string(), "warnings".to_string()]); -} +Add direct enum deserialization coverage: +```rust #[test] -fn bad_verbosity_string_fails_config_load() { - // Deserialize AuctionDebugCommentOptions directly, not a full Settings — - // Settings has required fields with no #[serde(default)] (e.g. - // `publisher`), so a full-Settings fixture missing them would fail with - // "missing field `publisher`" regardless of whether `verbosity` itself - // deserialized correctly, testing the wrong thing. - let result: Result = - toml::from_str(r#"verbosity = "everything""#); - assert!(result.is_err(), "unrecognized verbosity must fail to deserialize, not silently fall back"); +fn auction_debug_comment_options_deserializes_upstream_verbosity() { + let options: AuctionDebugCommentOptions = + toml::from_str("verbosity = \"upstream\"") + .expect("should deserialize upstream verbosity"); + assert_eq!(options.verbosity, AuctionDebugCommentVerbosity::Upstream); } ``` -Run: `cargo test -p trusted-server-core auction_debug_comment_options -- --nocapture` -Expected: FAIL to compile — `AuctionDebugCommentOptions`, `AuctionDebugCommentVerbosity`, `AUCTION_DEBUG_METADATA_ALLOWLIST` don't exist yet. +Change the normalization fixture to safe and unsafe-looking names so it proves normalization only, not authorization: + +```rust +let mut opts = AuctionDebugCommentOptions { + metadata_keys: vec![ + " http_status ".to_string(), + "".to_string(), + "debug".to_string(), + ], + ..AuctionDebugCommentOptions::default() +}; +opts.normalize(); +assert_eq!( + opts.metadata_keys, + vec!["http_status".to_string(), "debug".to_string()] +); +``` + +This deliberately leaves `debug` in normalized configuration; authorization must happen at render time. + +- [ ] **Step 2: Run the focused tests and confirm RED** -(Use `cargo test -p trusted-server-core`, not `cargo test-axum` — the latter is an alias for `cargo test -p trusted-server-adapter-axum` only per `.cargo/config.toml` and will NOT build or run `trusted-server-core`'s own `#[cfg(test)]` modules, silently reporting "0 passed; 0 failed" instead of actually exercising these tests. This applies to every test command in Task 1 and Task 3 below.) +Run: -- [ ] **Step 2: Implement the struct, enum, and allowlist constant** +```bash +cargo test -p trusted-server-core --lib auction_debug_comment_options -- --nocapture +``` -Insert into `settings.rs`, near the existing `DebugConfig` (around line 1894), replacing the old bool-only struct: +Expected: failure because the current default still contains provider-controlled keys and `AuctionDebugCommentVerbosity::Upstream` does not exist. + +- [ ] **Step 3: Implement the minimal settings change** + +Replace the current safe const with: ```rust -/// Metadata keys safe to surface in the `ts-debug` auction comment. -/// -/// Fail-closed superset: any key not listed here — notably `debug`, which -/// carries the resolved OpenRTB request (EC ID, `user.ext.eids`, the TC -/// consent string, `device.ip`, `device.geo`) plus per-bidder `httpcalls` — -/// is dropped in [`AuctionDebugCommentVerbosity::Redacted`] mode regardless -/// of what an operator lists in -/// [`AuctionDebugCommentOptions::metadata_keys`]. `metadata_keys` is a subset -/// selector against this const, never a way to add new keys. -pub(crate) const AUCTION_DEBUG_METADATA_ALLOWLIST: &[&str] = &[ - "error_type", - "http_status", - "message", - "upstream_message", - "upstream_message_truncated", - "responsetimemillis", +pub(crate) const AUCTION_DEBUG_METADATA_ALLOWLIST: &[&str] = + &["error_type", "http_status", "message"]; + +pub(crate) const AUCTION_DEBUG_UPSTREAM_METADATA_KEYS: &[&str] = &[ "errors", "warnings", + "responsetimemillis", "bidstatus", + "upstream_message", + "upstream_message_truncated", ]; +``` -fn default_true() -> bool { - true -} - -fn default_auction_debug_metadata_keys() -> Vec { - AUCTION_DEBUG_METADATA_ALLOWLIST - .iter() - .map(|s| s.to_string()) - .collect() -} - -/// Behavior of the `` auction dump. Only consulted when -/// [`DebugConfig::auction_html_comment`] is true. -/// -/// `deny_unknown_fields` matches the convention used by 17 of the ~19 -/// sibling config structs in this file, including the `DebugConfig` this -/// struct nests under: an operator typo (e.g. `metadata_key` instead of -/// `metadata_keys`) must fail config load loudly, not be silently ignored. -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct AuctionDebugCommentOptions { - /// Include the `provider_responses` section at all. - #[serde(default = "default_true")] - pub include_provider_responses: bool, - - /// Include `mediator_response` when a mediator ran. - #[serde(default = "default_true")] - pub include_mediator_response: bool, - - /// Include each provider's `bids` array (vs. status/metadata only). - #[serde(default = "default_true")] - pub include_bids: bool, - - /// Subset of [`AUCTION_DEBUG_METADATA_ALLOWLIST`] to surface in - /// [`AuctionDebugCommentVerbosity::Redacted`] mode. Keys outside the - /// fixed allowlist are always dropped, config or not. Ignored when - /// `verbosity` is `Full`. - #[serde(default = "default_auction_debug_metadata_keys")] - pub metadata_keys: Vec, - - /// `Redacted` (default): `metadata_keys` subset only, creative preview - /// truncated to `MAX_BID_CREATIVE_DUMP_BYTES`. - /// `Full`: raw `response.metadata` verbatim, including the `debug` - /// subtree (httpcalls/resolvedrequest) when present, and no creative - /// truncation. The total dump byte cap and comment-terminator - /// neutralization still apply unconditionally. - /// - /// NEVER enable `Full` in production — identity-bearing request/response - /// data becomes visible to any visitor via view-source. - #[serde(default)] - pub verbosity: AuctionDebugCommentVerbosity, -} - -impl Default for AuctionDebugCommentOptions { - fn default() -> Self { - Self { - include_provider_responses: true, - include_mediator_response: true, - include_bids: true, - metadata_keys: default_auction_debug_metadata_keys(), - verbosity: AuctionDebugCommentVerbosity::Redacted, - } - } -} - -impl AuctionDebugCommentOptions { - pub(crate) fn normalize(&mut self) { - self.metadata_keys = self - .metadata_keys - .drain(..) - .map(|key| key.trim().to_string()) - .filter(|key| !key.is_empty()) - .collect(); - } -} +Add the enum variant between `Redacted` and `Full`: -/// Verbosity of the `ts-debug` auction comment. See -/// [`AuctionDebugCommentOptions::verbosity`]. +```rust #[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum AuctionDebugCommentVerbosity { #[default] Redacted, + Upstream, Full, } ``` -Then update `DebugConfig` itself (replacing the doc comment on `auction_html_comment` isn't needed — it's unchanged — just add the new field after `auction_html_comment`): +Update the public docs on `metadata_keys` and `verbosity` to state: -```rust - #[serde(default)] - pub auction_html_comment: bool, - - /// Content and verbosity of the `auction_html_comment` dump. Ignored - /// when `auction_html_comment` is false. - #[serde(default)] - pub auction_html_comment_options: AuctionDebugCommentOptions, -``` +- `metadata_keys` selects only the three safe fields and cannot unlock upstream keys. +- `Upstream` adds six untyped provider diagnostic values and remains creative-truncated. +- `Full` copies all response metadata and does not truncate creatives. +- `Upstream` and `Full` must not be enabled in production. -- [ ] **Step 3: Wire normalize() into finalize_deserialized** - -In `settings.rs:2038-2059`, add one line after the existing normalize calls: - -```rust - pub(crate) fn finalize_deserialized( - mut settings: Self, - validation_label: &str, - ) -> Result> { - settings.integrations.normalize(); - settings.proxy.normalize(); - settings.image_optimizer.normalize(); - settings.debug.auction_html_comment_options.normalize(); - settings.consent.validate(); -``` +- [ ] **Step 4: Run focused tests and confirm GREEN** -- [ ] **Step 4: Run tests, verify pass** +Run the command from Step 2. -Run: `cargo test -p trusted-server-core auction_debug_comment_options -- --nocapture` -Expected: PASS (all 3 tests from Step 1) +Expected: all matching settings tests pass, including invalid-verbosity rejection. - [ ] **Step 5: Commit** ```bash git add crates/trusted-server-core/src/settings.rs -git commit -m "Add configurable options struct for the SSAT debug comment" +git commit -m "Harden SSAT debug comment configuration modes" ``` --- -### Task 2: Thread options through publisher.rs redaction functions +### Task 2: Make redacted rendering schema-safe **Files:** -- Modify: `crates/trusted-server-core/src/publisher.rs:870-936` (allowlist const removal, `redact_response_for_dump`, `redact_bid_for_dump`) -- Modify: `crates/trusted-server-core/src/publisher.rs:950-1036` (`prepend_auction_debug_comment`) -- Modify: `crates/trusted-server-core/src/publisher.rs:1394-1396` (production call site) -- Modify: `crates/trusted-server-core/src/publisher.rs:2638` and `:2699` (existing test call sites) +- Modify: `crates/trusted-server-core/src/publisher.rs:60-70` +- Modify: `crates/trusted-server-core/src/publisher.rs:1870-2035` +- Test: `crates/trusted-server-core/src/publisher.rs:4250-4565` -- [ ] **Step 1: Write failing tests for the new behavior** +- [ ] **Step 1: Add a reusable test renderer with explicit metadata** -Add to the `publisher.rs` test module, near the existing `auction_debug_comment_*` tests (~line 2648 onward). These replace the fixed `dump_comment_for_creative` helper's implicit "always default options" behavior with an explicit parameter, so update the helper first: +Near the existing `dump_comment_for_creative_with_options`, add a helper that builds one `AuctionResponse::error("prebid", 12)`, attaches supplied metadata with the existing `with_metadata` builder, calls `prepend_auction_debug_comment`, and returns the rendered state. Do not add a production convenience API only for tests. -```rust -/// Build the ts-debug comment for a one-bid auction whose creative is -/// `creative`, so tests can assert on the rendered dump. -fn dump_comment_for_creative_with_options( - creative: &str, - options: &AuctionDebugCommentOptions, -) -> String { - let mut bid = make_test_bid_with_creative(creative); - bid.slot_id = "ad-header-0".to_string(); - let result = OrchestrationResult { - provider_responses: vec![ - AuctionResponse::no_bid("prebid", 665), - AuctionResponse::success("aps", vec![bid], 42), - ], - mediator_response: None, - winning_bids: std::collections::HashMap::new(), - total_time_ms: 665, - metadata: std::collections::HashMap::new(), - }; - let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); - prepend_auction_debug_comment("stream", &result, &state, options); - let comment = state - .lock() - .expect("should lock state") - .clone() - .expect("should have comment"); - drop(state); - comment -} +- [ ] **Step 2: Write the failing default and subset tests** -fn dump_comment_for_creative(creative: &str) -> String { - dump_comment_for_creative_with_options(creative, &AuctionDebugCommentOptions::default()) -} +Replace `default_options_reproduce_current_behavior` with `default_options_apply_safe_response_metadata_schema`. Its fixture must contain: -#[test] -fn default_options_reproduce_current_behavior() { - // Identical to the pre-existing fixed output except: the unused `status` - // key (never written by any production path) is gone, and http_status / - // upstream_message / upstream_message_truncated are now allowlisted. - let comment = dump_comment_for_creative("
plain
"); - assert!(comment.contains("\"status\":\"nobid\"")); - assert!(comment.contains("dump={\"provider_responses\":")); - assert!(!comment.contains("mediator_response")); -} +- valid `error_type = "http_status"` +- valid `http_status = 422` +- malicious raw `message` +- all six upstream diagnostic keys containing unique fictional identity-shaped markers +- `debug.resolvedrequest.user.id` -#[test] -fn metadata_keys_empty_yields_empty_metadata_object() { - let options = AuctionDebugCommentOptions { - metadata_keys: vec![], - ..AuctionDebugCommentOptions::default() - }; - let comment = dump_comment_for_creative_with_options("
x
", &options); - assert!( - comment.contains("\"metadata\":{}"), - "empty metadata_keys should yield an empty metadata object: {comment}" - ); -} +Assert that output contains `error_type`, `http_status`, and the fixed message `Provider returned HTTP 422`; assert that none of the raw message, upstream markers, or `debug` subtree appear. -#[test] -fn metadata_keys_attack_vector_debug_key_never_surfaces_in_redacted_mode() { - // Configuring "debug" in metadata_keys must have zero effect in Redacted - // mode — the allowlist intersection is the actual security boundary, not - // the config value. This is the load-bearing test for this whole design. - let response = AuctionResponse::error("prebid", 12).with_metadata( - "debug", - serde_json::json!({"resolvedrequest": {"user": {"id": "EC-ID-abc123"}}}), - ); - let result = OrchestrationResult { - provider_responses: vec![response], - mediator_response: None, - winning_bids: std::collections::HashMap::new(), - total_time_ms: 12, - metadata: std::collections::HashMap::new(), - }; - let options = AuctionDebugCommentOptions { - metadata_keys: vec!["debug".to_string()], - ..AuctionDebugCommentOptions::default() - }; - let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); - prepend_auction_debug_comment("stream", &result, &state, &options); - let comment = state.lock().expect("should lock state").clone().expect("should have comment"); - assert!( - !comment.contains("EC-ID-abc123"), - "debug key must never surface in Redacted mode even if configured: {comment}" - ); -} +Add `configured_metadata_subset_only_includes_selected_safe_keys` with `metadata_keys = ["http_status", "errors", "debug"]`. Assert only validated `http_status` survives; `errors` and `debug` cannot be unlocked by selector configuration. -#[test] -fn verbosity_full_includes_raw_debug_subtree_when_present() { - let response = AuctionResponse::error("prebid", 12).with_metadata( - "debug", - serde_json::json!({"httpcalls": {"aps": [{"status": 200}]}}), - ); - let result = OrchestrationResult { - provider_responses: vec![response], - mediator_response: None, - winning_bids: std::collections::HashMap::new(), - total_time_ms: 12, - metadata: std::collections::HashMap::new(), - }; - let options = AuctionDebugCommentOptions { - verbosity: AuctionDebugCommentVerbosity::Full, - ..AuctionDebugCommentOptions::default() - }; - let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); - prepend_auction_debug_comment("stream", &result, &state, &options); - let comment = state.lock().expect("should lock state").clone().expect("should have comment"); - assert!( - comment.contains("httpcalls"), - "Full verbosity should surface the raw debug subtree: {comment}" - ); -} +- [ ] **Step 3: Write the failing adversarial schema test** -#[test] -fn verbosity_full_skips_creative_truncation() { - let big_creative = "y".repeat(MAX_BID_CREATIVE_DUMP_BYTES * 2); - let options = AuctionDebugCommentOptions { - verbosity: AuctionDebugCommentVerbosity::Full, - ..AuctionDebugCommentOptions::default() - }; - let comment = dump_comment_for_creative_with_options(&big_creative, &options); - assert!( - comment.contains(&big_creative), - "Full verbosity should not truncate the creative preview" - ); -} - -#[test] -fn verbosity_full_still_hits_overall_byte_cap() { - let huge_creative = "z".repeat(MAX_AUCTION_DEBUG_DUMP_BYTES * 2); - let options = AuctionDebugCommentOptions { - verbosity: AuctionDebugCommentVerbosity::Full, - ..AuctionDebugCommentOptions::default() - }; - let comment = dump_comment_for_creative_with_options(&huge_creative, &options); - assert!( - comment.contains("(truncated"), - "even Full verbosity must respect the total dump byte cap: {}", - &comment[..comment.len().min(200)] - ); -} +Add `redacted_mode_rejects_wrong_types_and_unknown_error_classifications`. Render separate cases containing: -#[test] -fn include_provider_responses_false_omits_section_entirely() { - let options = AuctionDebugCommentOptions { - include_provider_responses: false, - ..AuctionDebugCommentOptions::default() - }; - let comment = dump_comment_for_creative_with_options("
x
", &options); - assert!(!comment.contains("provider_responses")); -} - -#[test] -fn include_mediator_response_false_omits_even_when_mediator_ran() { - let response = AuctionResponse::success("aps", vec![], 10); - let mediator = AuctionResponse::success("mediator", vec![], 5); - let result = OrchestrationResult { - provider_responses: vec![response], - mediator_response: Some(mediator), - winning_bids: std::collections::HashMap::new(), - total_time_ms: 10, - metadata: std::collections::HashMap::new(), - }; - let options = AuctionDebugCommentOptions { - include_mediator_response: false, - ..AuctionDebugCommentOptions::default() - }; - let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); - prepend_auction_debug_comment("stream", &result, &state, &options); - let comment = state.lock().expect("should lock state").clone().expect("should have comment"); - assert!(!comment.contains("mediator_response")); -} +```rust +json!({"error_type": {"identity": "example-user-123"}}) +json!({"error_type": "provider_supplied_unknown", "message": "example-user-123"}) +json!({"http_status": "200 example-user-123"}) +json!({"http_status": 99}) +json!({"http_status": 600}) +json!({"message": {"identity": "example-user-123"}}) +``` -#[test] -fn include_bids_false_yields_empty_bids_array_not_omitted_response() { - let options = AuctionDebugCommentOptions { - include_bids: false, - ..AuctionDebugCommentOptions::default() - }; - let comment = dump_comment_for_creative_with_options("
x
", &options); - assert!(comment.contains("\"bids\":[]")); - // The provider entry itself (status/provider name) must still be present. - assert!(comment.contains("\"provider\":\"aps\"")); -} +Assert no identity marker or attacker-provided message appears. Also assert valid integer boundaries `100` and `599` survive, while non-integral JSON numbers do not. -#[test] -fn verbosity_full_still_neutralises_comment_terminators() { - let options = AuctionDebugCommentOptions { - verbosity: AuctionDebugCommentVerbosity::Full, - ..AuctionDebugCommentOptions::default() - }; - for creative in ["
evil-->break
", "--!>"] { - let comment = dump_comment_for_creative_with_options(creative, &options); - assert_eq!(comment.matches("-->").count(), 1); - assert!(!comment.contains("--!>")); - } -} -``` +- [ ] **Step 4: Write the failing safe-message mapping test** -Also update the two pre-existing tests that call `prepend_auction_debug_comment` directly with the old 3-arg signature — `auction_debug_comment_dumps_provider_status` (uses the helper, already covered by the helper update above) and `auction_debug_comment_never_leaks_provider_debug_metadata` (~line 2699, calls the function directly): +Add a table-driven test for the fixed mappings: ```rust - let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); - prepend_auction_debug_comment("stream", &result, &state, &AuctionDebugCommentOptions::default()); +let cases = [ + ("parse_response", None, "Provider response could not be parsed"), + ("launch_failed", None, "Provider launch failed"), + ("transport", None, "Provider request failed"), + ("timeout", None, "Provider request timed out"), + ("http_status", Some(418), "Provider returned HTTP 418"), + ("http_status", None, "Provider returned an HTTP error"), +]; ``` -Run: `cargo test-axum -p trusted-server-core --lib publisher:: -- --nocapture` -Expected: FAIL to compile — `prepend_auction_debug_comment` doesn't take a 4th argument yet; `redact_response_for_dump`/`redact_bid_for_dump` don't take `options` yet. +For every case, include a malicious raw `metadata["message"]` and prove the renderer ignores it. -- [ ] **Step 2: Remove the old local allowlist const, import the new one** +- [ ] **Step 5: Run the focused publisher tests and confirm RED** -Delete from `publisher.rs` (lines 870-886, the old `DEBUG_DUMP_METADATA_ALLOWLIST` const and its doc comment) and add near the top of the file's imports: +Run: -```rust -use crate::settings::AUCTION_DEBUG_METADATA_ALLOWLIST; -use crate::settings::{AuctionDebugCommentOptions, AuctionDebugCommentVerbosity}; +```bash +cargo test -p trusted-server-core --lib publisher::tests:: -- --nocapture ``` -- [ ] **Step 3: Update redact_bid_for_dump and redact_response_for_dump** +Expected: new privacy tests fail because current redacted rendering clones values by matching key names only. + +- [ ] **Step 6: Implement schema validation and safe message generation** -Replace the two functions (publisher.rs ~905-936): +Import both key constants. Add private helpers close to `redact_response_for_dump`: ```rust -/// Build a redacted JSON view of a single provider response for the -/// `ts-debug` dump. In [`AuctionDebugCommentVerbosity::Redacted`], only keys -/// in `options.metadata_keys ∩ AUCTION_DEBUG_METADATA_ALLOWLIST` survive and -/// each bid's creative is previewed to [`MAX_BID_CREATIVE_DUMP_BYTES`]. In -/// [`AuctionDebugCommentVerbosity::Full`], metadata and creatives pass -/// through unfiltered. -fn redact_response_for_dump( - response: &crate::auction::types::AuctionResponse, - options: &AuctionDebugCommentOptions, -) -> serde_json::Value { - let metadata: serde_json::Map = match options.verbosity { - AuctionDebugCommentVerbosity::Redacted => response - .metadata - .iter() - .filter(|(key, _)| { - options.metadata_keys.iter().any(|configured| configured == *key) - && AUCTION_DEBUG_METADATA_ALLOWLIST.contains(&key.as_str()) - }) - .map(|(key, value)| (key.clone(), value.clone())) - .collect(), - AuctionDebugCommentVerbosity::Full => response - .metadata - .iter() - .map(|(key, value)| (key.clone(), value.clone())) - .collect(), - }; - let bids: Vec = if options.include_bids { - response.bids.iter().map(|bid| redact_bid_for_dump(bid, options)).collect() - } else { - Vec::new() - }; - serde_json::json!({ - "provider": response.provider, - "status": response.status, - "response_time_ms": response.response_time_ms, - "bids": bids, - "metadata": metadata, - }) +fn validated_error_type(metadata: &serde_json::Map) -> Option<&str> { + let value = metadata.get("error_type")?.as_str()?; + matches!( + value, + "parse_response" | "launch_failed" | "transport" | "timeout" | "http_status" + ) + .then_some(value) } -/// Build a redacted JSON view of a single bid. In `Redacted` verbosity, -/// `creative` is previewed to [`MAX_BID_CREATIVE_DUMP_BYTES`]; in `Full`, it -/// passes through untruncated. -fn redact_bid_for_dump( - bid: &crate::auction::types::Bid, - options: &AuctionDebugCommentOptions, -) -> serde_json::Value { - let mut value = serde_json::to_value(bid).unwrap_or(serde_json::Value::Null); - if options.verbosity == AuctionDebugCommentVerbosity::Redacted - && let Some(creative) = &bid.creative - { - value["creative"] = - serde_json::Value::String(truncate_with_marker(creative, MAX_BID_CREATIVE_DUMP_BYTES)); +fn validated_http_status( + metadata: &serde_json::Map, +) -> Option { + metadata + .get("http_status")? + .as_u64() + .filter(|status| (100..=599).contains(status)) +} + +fn safe_error_message(error_type: &str, http_status: Option) -> Option { + match error_type { + "parse_response" => Some("Provider response could not be parsed".to_string()), + "launch_failed" => Some("Provider launch failed".to_string()), + "transport" => Some("Provider request failed".to_string()), + "timeout" => Some("Provider request timed out".to_string()), + "http_status" => Some(http_status.map_or_else( + || "Provider returned an HTTP error".to_string(), + |status| format!("Provider returned HTTP {status}"), + )), + _ => None, } - value } ``` -Note the `metadata_keys.iter().any(...)` check: this is the intersection — a key must be BOTH configured AND in the hardcoded superset. `AUCTION_DEBUG_METADATA_ALLOWLIST.contains` alone would let an operator narrow but a bug in this line (e.g. only checking `metadata_keys`) would break the fail-closed guarantee. This is exactly what `metadata_keys_attack_vector_debug_key_never_surfaces_in_redacted_mode` (Task 2, Step 1) verifies. - -- [ ] **Step 4: Update prepend_auction_debug_comment** - -Replace the function body (publisher.rs ~950-1028) to add the `options` parameter and gate the two top-level sections: +Implement `redacted_metadata_for_dump` so it reconstructs values only when each field is configured and valid. It must never clone `metadata["message"]`: ```rust -pub(crate) fn prepend_auction_debug_comment( - path_label: &str, - result: &crate::auction::orchestrator::OrchestrationResult, - ad_bids_state: &Arc>>, +fn redacted_metadata_for_dump( + metadata: &serde_json::Map, options: &AuctionDebugCommentOptions, -) { - let ssp_count = result.provider_responses.len(); - let mediator_info = match &result.mediator_response { - Some(r) => format!("ok({}_bids)", r.bids.len()), - None => "none".to_string(), +) -> serde_json::Map { + let selected = |key: &str| { + AUCTION_DEBUG_METADATA_ALLOWLIST.contains(&key) + && options.metadata_keys.iter().any(|candidate| candidate == key) }; - let mut dump = serde_json::Map::new(); - if options.include_provider_responses { - dump.insert( - "provider_responses".to_string(), - serde_json::Value::Array( - result - .provider_responses - .iter() - .map(|r| redact_response_for_dump(r, options)) - .collect(), - ), - ); + let error_type = validated_error_type(metadata); + let http_status = validated_http_status(metadata); + let mut safe = serde_json::Map::new(); + if selected("error_type") && let Some(value) = error_type { + safe.insert("error_type".to_string(), serde_json::json!(value)); + } + if selected("http_status") && let Some(value) = http_status { + safe.insert("http_status".to_string(), serde_json::json!(value)); } - if options.include_mediator_response - && let Some(mediator_response) = &result.mediator_response + if selected("message") + && let Some(value) = error_type.and_then(|kind| safe_error_message(kind, http_status)) { - dump.insert( - "mediator_response".to_string(), - redact_response_for_dump(mediator_response, options), - ); + safe.insert("message".to_string(), serde_json::json!(value)); } - // ... rest of the function (render_dump closure, debug_comment format!, - // state locking) is UNCHANGED — do not modify below this point. + safe +} ``` -Everything from the `render_dump` closure onward (the neutralization + byte-cap logic, the `format!("` HTML comment before `` dumping a - /// redacted per-provider auction result: pipeline stats (SSP count, mediator - /// status, winning bid count) plus every provider response — each bid's - /// creative previewed (not the full `adm` markup) and provider metadata - /// filtered to a fail-closed allowlist that drops identity-bearing keys. - /// Never enable in production — visible in page source and injects (bounded) - /// raw HTML from SSPs. + /// Inject a `` HTML comment before `` dumping + /// per-provider auction diagnostics. The default validates response-level + /// metadata, but bid fields and bounded creative previews remain visible; + /// this is not a fully anonymized dump. Never enable in production. #[serde(default)] pub auction_html_comment: bool, @@ -1917,16 +1914,21 @@ pub struct DebugConfig { /// of what an operator lists in [`AuctionDebugCommentOptions::metadata_keys`]. /// `metadata_keys` is a subset selector against this const, never a way to /// add new keys. -pub(crate) const AUCTION_DEBUG_METADATA_ALLOWLIST: &[&str] = &[ - "error_type", - "http_status", - "message", - "upstream_message", - "upstream_message_truncated", - "responsetimemillis", +pub(crate) const AUCTION_DEBUG_METADATA_ALLOWLIST: &[&str] = + &["error_type", "http_status", "message"]; + +/// Provider-controlled diagnostic keys exposed only by `Upstream` or `Full`. +/// +/// Values remain untyped upstream JSON and may contain request or identity +/// data. Keeping this list separate prevents [`AuctionDebugCommentOptions::metadata_keys`] +/// from widening the default response-metadata boundary. +pub(crate) const AUCTION_DEBUG_UPSTREAM_METADATA_KEYS: &[&str] = &[ "errors", "warnings", + "responsetimemillis", "bidstatus", + "upstream_message", + "upstream_message_truncated", ]; fn default_true() -> bool { @@ -1964,20 +1966,22 @@ pub struct AuctionDebugCommentOptions { /// Subset of [`AUCTION_DEBUG_METADATA_ALLOWLIST`] to surface in /// [`AuctionDebugCommentVerbosity::Redacted`] mode. Keys outside the - /// fixed allowlist are always dropped, config or not. Ignored when - /// `verbosity` is `Full`. + /// fixed allowlist are always dropped, config or not. This selector cannot + /// unlock provider diagnostics. Ignored when `verbosity` is `Full`. #[serde(default = "default_auction_debug_metadata_keys")] pub metadata_keys: Vec, - /// `Redacted` (default): `metadata_keys` subset only, creative preview - /// truncated to `MAX_BID_CREATIVE_DUMP_BYTES`. + /// `Redacted` (default): validated `metadata_keys` subset only, with + /// creative previews truncated to `MAX_BID_CREATIVE_DUMP_BYTES`. + /// `Upstream`: redacted fields plus six untyped provider diagnostics; + /// creative previews remain truncated. /// `Full`: raw `response.metadata` verbatim, including the `debug` /// subtree (httpcalls/resolvedrequest) when present, and no creative /// truncation. The total dump byte cap and comment-terminator /// neutralization still apply unconditionally. /// - /// NEVER enable `Full` in production — identity-bearing request/response - /// data becomes visible to any visitor via view-source. + /// NEVER enable `Upstream` or `Full` in production — identity-bearing + /// request/response data may become visible via view-source. #[serde(default)] pub verbosity: AuctionDebugCommentVerbosity, } @@ -2012,6 +2016,7 @@ impl AuctionDebugCommentOptions { pub enum AuctionDebugCommentVerbosity { #[default] Redacted, + Upstream, Full, } @@ -2716,11 +2721,12 @@ mod tests { assert!(opts.include_bids, "should default to true"); assert_eq!( opts.metadata_keys, - AUCTION_DEBUG_METADATA_ALLOWLIST - .iter() - .map(std::string::ToString::to_string) - .collect::>(), - "should default metadata_keys to the full allowlist" + vec![ + "error_type".to_string(), + "http_status".to_string(), + "message".to_string(), + ], + "should default to only schema-validated response metadata" ); assert_eq!( opts.verbosity, @@ -2733,19 +2739,26 @@ mod tests { fn auction_debug_comment_options_normalize_trims_and_drops_empty_keys() { let mut opts = AuctionDebugCommentOptions { metadata_keys: vec![ - " status ".to_string(), + " http_status ".to_string(), "".to_string(), - "warnings".to_string(), + "debug".to_string(), ], ..AuctionDebugCommentOptions::default() }; opts.normalize(); assert_eq!( opts.metadata_keys, - vec!["status".to_string(), "warnings".to_string()] + vec!["http_status".to_string(), "debug".to_string()] ); } + #[test] + fn auction_debug_comment_options_deserializes_upstream_verbosity() { + let options: AuctionDebugCommentOptions = toml::from_str(r#"verbosity = "upstream""#) + .expect("should deserialize upstream verbosity"); + assert_eq!(options.verbosity, AuctionDebugCommentVerbosity::Upstream); + } + #[test] fn bad_verbosity_string_fails_config_load() { // Deserialize AuctionDebugCommentOptions directly, not a full Settings — From 567c9632fabe550d98618f5bb12fa6d147309dc7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 21:48:58 +0530 Subject: [PATCH 277/395] Enforce SSAT debug response metadata schemas --- crates/trusted-server-core/src/publisher.rs | 495 +++++++++++++++++--- 1 file changed, 418 insertions(+), 77 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index cf42db830..f808ea85b 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -62,8 +62,8 @@ use crate::price_bucket::{PriceGranularity, price_bucket}; use crate::response_privacy::enforce_synthesized_html_cache_privacy; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::{ - AUCTION_DEBUG_METADATA_ALLOWLIST, AuctionDebugCommentOptions, AuctionDebugCommentVerbosity, - Settings, + AUCTION_DEBUG_METADATA_ALLOWLIST, AUCTION_DEBUG_UPSTREAM_METADATA_KEYS, + AuctionDebugCommentOptions, AuctionDebugCommentVerbosity, Settings, }; use crate::streaming_processor::{ BodyStreamDecoder, BodyStreamEncoder, Compression, GzipDecodeReader, PipelineConfig, @@ -1891,29 +1891,99 @@ fn truncate_with_marker(value: &str, max: usize) -> String { format!("{}…(truncated {} bytes)", &value[..end], value.len() - end) } -/// Build a redacted JSON view of a single provider response for the -/// `ts-debug` dump. In [`AuctionDebugCommentVerbosity::Redacted`], only keys -/// in `options.metadata_keys ∩ AUCTION_DEBUG_METADATA_ALLOWLIST` survive and -/// each bid's creative is previewed to [`MAX_BID_CREATIVE_DUMP_BYTES`]. In -/// [`AuctionDebugCommentVerbosity::Full`], metadata and creatives pass -/// through unfiltered. +/// Return a recognized server-owned provider error classification. +fn validated_error_type( + metadata: &std::collections::HashMap, +) -> Option<&str> { + let value = metadata.get("error_type")?.as_str()?; + matches!( + value, + "parse_response" | "launch_failed" | "transport" | "timeout" | "http_status" + ) + .then_some(value) +} + +/// Return a valid HTTP response status from provider metadata. +fn validated_http_status( + metadata: &std::collections::HashMap, +) -> Option { + metadata + .get("http_status")? + .as_u64() + .filter(|status| (100..=599).contains(status)) +} + +/// Generate public diagnostic wording without copying provider-controlled text. +fn safe_error_message(error_type: &str, http_status: Option) -> Option { + match error_type { + "parse_response" => Some("Provider response could not be parsed".to_string()), + "launch_failed" => Some("Provider launch failed".to_string()), + "transport" => Some("Provider request failed".to_string()), + "timeout" => Some("Provider request timed out".to_string()), + "http_status" => Some(http_status.map_or_else( + || "Provider returned an HTTP error".to_string(), + |status| format!("Provider returned HTTP {status}"), + )), + _ => None, + } +} + +/// Reconstruct the configured response metadata from validated values. +fn redacted_metadata_for_dump( + metadata: &std::collections::HashMap, + options: &AuctionDebugCommentOptions, +) -> serde_json::Map { + let selected = |key: &str| { + AUCTION_DEBUG_METADATA_ALLOWLIST.contains(&key) + && options + .metadata_keys + .iter() + .any(|candidate| candidate == key) + }; + let error_type = validated_error_type(metadata); + let http_status = validated_http_status(metadata); + let mut safe = serde_json::Map::new(); + + if selected("error_type") + && let Some(value) = error_type + { + safe.insert("error_type".to_string(), serde_json::json!(value)); + } + if selected("http_status") + && let Some(value) = http_status + { + safe.insert("http_status".to_string(), serde_json::json!(value)); + } + if selected("message") + && let Some(value) = error_type.and_then(|kind| safe_error_message(kind, http_status)) + { + safe.insert("message".to_string(), serde_json::json!(value)); + } + + safe +} + +/// Build a JSON view of a single provider response for the `ts-debug` dump. +/// +/// `Redacted` reconstructs only schema-validated response metadata, `Upstream` +/// adds six named provider diagnostics, and `Full` copies every metadata value. fn redact_response_for_dump( response: &crate::auction::types::AuctionResponse, options: &AuctionDebugCommentOptions, ) -> serde_json::Value { let metadata: serde_json::Map = match options.verbosity { - AuctionDebugCommentVerbosity::Redacted | AuctionDebugCommentVerbosity::Upstream => response - .metadata - .iter() - .filter(|(key, _)| { - options - .metadata_keys - .iter() - .any(|configured| configured == *key) - && AUCTION_DEBUG_METADATA_ALLOWLIST.contains(&key.as_str()) - }) - .map(|(key, value)| (key.clone(), value.clone())) - .collect(), + AuctionDebugCommentVerbosity::Redacted => { + redacted_metadata_for_dump(&response.metadata, options) + } + AuctionDebugCommentVerbosity::Upstream => { + let mut metadata = redacted_metadata_for_dump(&response.metadata, options); + for key in AUCTION_DEBUG_UPSTREAM_METADATA_KEYS { + if let Some(value) = response.metadata.get(*key) { + metadata.insert((*key).to_string(), value.clone()); + } + } + metadata + } AuctionDebugCommentVerbosity::Full => response .metadata .iter() @@ -1938,9 +2008,8 @@ fn redact_response_for_dump( }) } -/// Build a redacted JSON view of a single bid. In `Redacted` verbosity, -/// `creative` is previewed to [`MAX_BID_CREATIVE_DUMP_BYTES`]; in `Full`, it -/// passes through untruncated. +/// Build a JSON view of a single bid. `Redacted` and `Upstream` preview the +/// creative to [`MAX_BID_CREATIVE_DUMP_BYTES`]; `Full` passes it through. fn redact_bid_for_dump( bid: &crate::auction::types::Bid, options: &AuctionDebugCommentOptions, @@ -1959,9 +2028,11 @@ fn redact_bid_for_dump( /// auction result — pipeline stats plus, per provider, its status, bids, and /// metadata, shaped by `options` — onto the shared `ad_bids_state` so it /// lands directly before the injected bids `", + "", escaped ) } @@ -3635,14 +3394,12 @@ pub(crate) fn build_empty_bids_script() -> String { /// [`handle_page_bids`] (SPA navigation) so the slot wire shape has a single /// definition and the two paths cannot silently diverge. Property names match /// what the client-side TSJS bundle expects: `gam_unit_path`, `div_id`, -/// `formats`, and `targeting`. Returns `None` when the slot's dynamic GAM unit -/// path exceeds its rendering limit. -pub(crate) fn build_slot_json( +/// `formats`, and `targeting`. +fn build_slot_json( slot: &crate::creative_opportunities::CreativeOpportunitySlot, co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, - section: &str, -) -> Option { - let gam_path = slot.render_gam_unit_path(&co_config.gam_network_id, section)?; +) -> serde_json::Value { + let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); let div_id = slot.resolved_div_id(); let formats: Vec = slot .formats @@ -3654,40 +3411,13 @@ pub(crate) fn build_slot_json( .iter() .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) .collect(); - Some(serde_json::json!({ + serde_json::json!({ "id": slot.id, "gam_unit_path": gam_path, "div_id": div_id, "formats": formats, "targeting": targeting, - })) -} - -/// Match creative-opportunity slots and omit dynamic GAM paths that cannot be -/// rendered for this request before they can enter an auction. -fn match_renderable_slots( - slots: &[crate::creative_opportunities::CreativeOpportunitySlot], - co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, - request_path: &str, -) -> Vec { - let section = co_config.section_for_path(request_path); - crate::creative_opportunities::match_slots(slots, request_path) - .into_iter() - .filter_map(|slot| { - if slot - .render_gam_unit_path(&co_config.gam_network_id, §ion) - .is_none() - { - log::warn!( - "Omitting slot `{}`: dynamic gam_unit_path exceeds the render limit for path `{}`", - slot.id, - request_path - ); - return None; - } - Some(slot.clone()) - }) - .collect() + }) } /// Build the `tsjs.adSlots` `"); @@ -8767,91 +7622,6 @@ mod tests { assert!(!inner.contains('>'), "no unescaped > in script content"); } - #[test] - fn ad_slots_script_omits_only_over_limit_dynamic_slot() { - let mut over_limit = make_slot(); - over_limit.id = "over_limit_dynamic".to_string(); - over_limit.gam_unit_path = Some("/{section}/{section}".to_string()); - over_limit - .compile_unit_template() - .expect("template should compile"); - let mut valid_static = make_slot(); - valid_static.id = "valid_static_sibling".to_string(); - valid_static.gam_unit_path = Some("/12345/example/static".to_string()); - let slots = vec![over_limit, valid_static]; - let config = make_config(); - let request_path = format!("/{}", "a".repeat(60)); - - let script = build_ad_slots_script(&slots, &config, &request_path); - - assert!( - !script.contains("over_limit_dynamic"), - "should omit the over-limit dynamic slot" - ); - assert!( - script.contains("valid_static_sibling"), - "should retain the valid static sibling" - ); - } - - #[test] - fn build_slot_json_renders_section_from_request_path() { - let mut config = make_config(); - config.gam_network_id = "99999".to_string(); - config.section_root = Some("homepage".to_string()); - let mut slot = make_slot(); - slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); - slot.compile_unit_template() - .expect("template should compile"); - - let news_section = config.section_for_path("/news/article-123"); - let news = crate::publisher::build_slot_json(&slot, &config, &news_section) - .expect("should render slot"); - assert_eq!( - news["gam_unit_path"], "/99999/example/news", - "section should derive from the first path segment" - ); - - let home_section = config.section_for_path("/"); - let home = crate::publisher::build_slot_json(&slot, &config, &home_section) - .expect("should render slot"); - assert_eq!( - home["gam_unit_path"], "/99999/example/homepage", - "root path should use section_root" - ); - } - - #[test] - fn build_slot_json_honours_configured_section_segment() { - // Locale-prefixed publisher: `/en/news/article` must resolve to the - // `news` unit, not `en`. - let mut config = make_config(); - config.gam_network_id = "99999".to_string(); - config.section_root = Some("homepage".to_string()); - config.section_segment = Some(1); - let mut slot = make_slot(); - slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); - slot.compile_unit_template() - .expect("template should compile"); - - let news_section = config.section_for_path("/en/news/article-123"); - let news = crate::publisher::build_slot_json(&slot, &config, &news_section) - .expect("should render slot"); - assert_eq!( - news["gam_unit_path"], "/99999/example/news", - "section should derive from the configured segment index" - ); - - let locale_root_section = config.section_for_path("/en"); - let locale_root = - crate::publisher::build_slot_json(&slot, &config, &locale_root_section) - .expect("should render slot"); - assert_eq!( - locale_root["gam_unit_path"], "/99999/example/homepage", - "a path with no segment at the configured index should use section_root" - ); - } - #[test] fn bid_map_includes_nurl_and_burl() { let mut winning_bids = HashMap::new(); @@ -8902,139 +7672,6 @@ mod tests { ); } - /// Guards the browser-visible token every auction path shares: it must - /// be fresh per auction and absent unless diagnostics can consume it. - #[test] - fn diagnostics_auction_id_is_fresh_and_gated() { - let mut settings = test_settings(); - assert_eq!( - diagnostics_auction_id(&settings), - None, - "no token should be minted without the diagnostics integration" - ); - - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); - let first = - diagnostics_auction_id(&settings).expect("enabled diagnostics should mint a token"); - let second = - diagnostics_auction_id(&settings).expect("enabled diagnostics should mint a token"); - - assert!( - first.starts_with("ts-auc-"), - "token should use the diagnostics prefix, got `{first}`" - ); - assert_ne!(first, second, "each auction should mint its own token"); - } - - #[test] - fn initial_document_bids_script_includes_auction_id_only_for_winning_bids() { - let slot = make_slot(); - let slots = [slot]; - let slots_ctx = MatchedSlotsContext { - matched_slots: &slots, - request_path: "/2024/01/my-article/", - }; - let request_info = RequestInfo { - host: "publisher.example.com".to_string(), - scheme: "https".to_string(), - }; - let mut auction_request = build_auction_request( - &slots_ctx, - None, - &ConsentContext::default(), - &request_info, - "publisher.example.com", - Some("Mozilla/5.0"), - ); - auction_request.id = "initial-auction-example-123".to_string(); - let mut winning_bids = HashMap::new(); - winning_bids.insert( - "atf_sidebar_ad".to_string(), - make_bid( - "atf_sidebar_ad", - 1.50, - "example_bidder", - "abc123", - "https://example.com/win", - "https://example.com/bill", - ), - ); - - let state = std::sync::Arc::new(std::sync::Mutex::new(None)); - write_bids_to_state( - &winning_bids, - PriceGranularity::Dense, - &state, - &test_settings(), - "", - false, - Some(&auction_request.id), - ); - let script = state - .lock() - .expect("should lock initial bid state") - .clone() - .expect("should generate initial-document bids script"); - let bid_json = script - .strip_prefix( - "", - ) - }) - .expect("should emit the initial-document tsjs.bids script shape"); - let bid_json: String = serde_json::from_str(&format!("\"{bid_json}\"")) - .expect("should decode initial-document JSON.parse input"); - let bids: serde_json::Value = serde_json::from_str(&bid_json) - .expect("should serialize initial-document bids as JSON"); - - assert_eq!( - bids["atf_sidebar_ad"]["hb_auction_id"], auction_request.id, - "initial-document bids should expose the current request ID only on the winner" - ); - - write_bids_to_state( - &HashMap::new(), - PriceGranularity::Dense, - &state, - &test_settings(), - "", - false, - Some(&auction_request.id), - ); - let empty_script = state - .lock() - .expect("should lock empty initial bid state") - .clone() - .expect("should generate empty initial-document bids script"); - let empty_bid_json = empty_script - .strip_prefix( - "", - ) - }) - .expect("should emit the empty initial-document tsjs.bids script shape"); - let empty_bid_json: String = serde_json::from_str(&format!("\"{empty_bid_json}\"")) - .expect("should decode empty initial-document JSON.parse input"); - let empty_bids: serde_json::Value = serde_json::from_str(&empty_bid_json) - .expect("should serialize empty initial-document bids as JSON"); - assert!( - empty_bids - .as_object() - .expect("initial-document bids should be an object") - .is_empty(), - "initial-document bids should not fabricate metadata without a winner" - ); - } - #[test] fn bid_map_omits_zero_creative_dimensions() { // Missing OpenRTB w/h parse to 0. Emitting w:0/h:0 would make the @@ -9152,13 +7789,10 @@ mod tests { #[test] fn build_bid_map_sanitizes_hostile_adm() { - // The inline-adm path must run the same opt-in creative-processing - // boundary as the `/auction` path (sanitize → rewrite) before the - // creative reaches window.tsjs.bids, so with sanitization enabled - // hostile executable markup never lands in the client-facing `adm` - // for the Prebid Universal Creative to run. - let mut settings = test_settings(); - settings.auction.sanitize_creatives = true; + // The inline-adm path must run the same creative-processing boundary + // as the `/auction` path (sanitize → rewrite) before the creative + // reaches window.tsjs.bids, so hostile executable markup never lands + // in the client-facing `adm` for the Prebid Universal Creative to run. let mut winning_bids = HashMap::new(); let mut bid = make_bid( "atf_sidebar_ad", @@ -9175,7 +7809,13 @@ mod tests { ); winning_bids.insert("atf_sidebar_ad".to_string(), bid); - let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); + let map = build_bid_map( + &winning_bids, + PriceGranularity::Dense, + &test_settings(), + "", + false, + ); let adm = map .get("atf_sidebar_ad") .and_then(|v| v.as_object()) @@ -9202,9 +7842,8 @@ mod tests { } #[test] - fn build_bid_map_can_skip_rewriting_while_sanitizing() { + fn build_bid_map_can_skip_rewriting_but_not_sanitization() { let mut settings = test_settings(); - settings.auction.sanitize_creatives = true; settings.auction.rewrite_creatives = false; let mut winning_bids = HashMap::new(); let mut bid = make_bid( @@ -9261,11 +7900,10 @@ mod tests { #[test] fn build_bid_map_omits_oversized_adm() { - // Creatives larger than the 1 MiB cap are rejected (empty result) - // in every processing mode, so the bid is omitted rather than - // recording a blank winner or shipping an unbounded creative to the - // client. Runs with default settings to cover the shipped - // configuration. + // Creatives larger than the sanitize pass's 1 MiB cap are rejected + // (empty result), so the inline `adm` is omitted and the pbRender + // bridge falls back to the PBS Cache coordinates instead of shipping + // an unbounded creative to the client. let mut winning_bids = HashMap::new(); let mut bid = make_bid( "atf_sidebar_ad", @@ -9278,110 +7916,6 @@ mod tests { bid.creative = Some(format!("
{}
", "a".repeat(1024 * 1024 + 1))); winning_bids.insert("atf_sidebar_ad".to_string(), bid); - let map = build_bid_map( - &winning_bids, - PriceGranularity::Dense, - &test_settings(), - "", - false, - ); - assert!( - !map.contains_key("atf_sidebar_ad"), - "should omit the bid when the creative exceeds the 1 MiB cap" - ); - } - - // A supplied creative that processing rejects must not fall back to the - // PBS Cache coordinates: the GPT bridge fetches the cached bid's ORIGINAL - // adm, which would undo sanitization and the size cap entirely. - fn cached_bid_with_creative(creative: &str) -> Bid { - Bid { - slot_id: "atf_sidebar_ad".to_string(), - price: Some(1.50), - currency: "USD".to_string(), - creative: Some(creative.to_string()), - adomain: None, - bidder: "prebid".to_string(), - width: 300, - height: 250, - nurl: None, - burl: None, - ad_id: Some("bid-impression-id".to_string()), - bid_id: Some("openrtb-bid-id".to_string()), - creative_id: None, - // No typed renderer: these cases assert what happens when the - // supplied markup is the bid's only render source. - renderer: None, - cache_id: Some("cache-uuid".to_string()), - cache_host: Some("prebid-cache.example.com".to_string()), - cache_path: Some("/cache".to_string()), - metadata: Default::default(), - } - } - - // These fixtures carry cache coordinates but no typed renderer, so a - // rejected creative leaves the bid with no render source at all and it - // is dropped outright — which subsumes the property under test: the - // cache coordinates never reach the client, so the cached (unprocessed) - // copy of the markup cannot be fetched in place of what was refused. - fn assert_no_render_source(settings: &Settings, creative: String, case: &str) { - let mut winning_bids = HashMap::new(); - let mut bid = cached_bid_with_creative(""); - bid.creative = Some(creative); - winning_bids.insert("atf_sidebar_ad".to_string(), bid); - - let map = build_bid_map(&winning_bids, PriceGranularity::Dense, settings, "", false); - - match map.get("atf_sidebar_ad").and_then(|v| v.as_object()) { - None => {} - Some(obj) => { - assert!( - obj.get("adm").is_none(), - "{case}: rejected creative should not emit adm" - ); - assert!( - obj.get("hb_cache_host").is_none(), - "{case}: rejected creative should suppress hb_cache_host" - ); - assert!( - obj.get("hb_cache_path").is_none(), - "{case}: rejected creative should suppress hb_cache_path" - ); - } - } - } - - #[test] - fn build_bid_map_suppresses_cache_fallback_for_rejected_creatives() { - let mut sanitizing = test_settings(); - sanitizing.auction.sanitize_creatives = true; - - // Script-only creative: sanitization strips everything. - assert_no_render_source( - &sanitizing, - "".to_string(), - "script-only", - ); - // Oversized creative: rejected by the cap in every mode. - assert_no_render_source( - &test_settings(), - format!("
{}
", "a".repeat(1024 * 1024 + 1)), - "oversized", - ); - // An explicit empty `adm` is a supplied creative, not an absent one: - // classifying it as absent would re-enable the raw cache fallback. - assert_no_render_source(&test_settings(), String::new(), "explicit-empty"); - } - - #[test] - fn build_bid_map_keeps_cache_fallback_for_absent_creatives() { - // A bid with no supplied creative is the legitimate PBS Cache case: - // the coordinates are the only render source. - let mut winning_bids = HashMap::new(); - let mut bid = cached_bid_with_creative(""); - bid.creative = None; - winning_bids.insert("atf_sidebar_ad".to_string(), bid); - let map = build_bid_map( &winning_bids, PriceGranularity::Dense, @@ -9393,16 +7927,9 @@ mod tests { .get("atf_sidebar_ad") .and_then(|v| v.as_object()) .expect("should have a bid entry"); - - assert_eq!( - obj.get("hb_cache_host").and_then(|v| v.as_str()), - Some("prebid-cache.example.com"), - "absent creative should keep hb_cache_host" - ); - assert_eq!( - obj.get("hb_cache_path").and_then(|v| v.as_str()), - Some("/cache"), - "absent creative should keep hb_cache_path" + assert!( + obj.get("adm").is_none(), + "should omit the inline adm when the creative exceeds the 1 MiB cap" ); } @@ -9414,7 +7941,6 @@ mod tests { // root-relative `/first-party/proxy` would resolve against GAM and 404. // The tsjs bundle must NOT be injected into that foreign-origin iframe. let mut settings = test_settings(); - settings.auction.rewrite_creatives = true; settings.publisher.domain = "example.com".to_string(); let mut winning_bids = HashMap::new(); @@ -9464,7 +7990,6 @@ mod tests { // origin the visitor is on (here an HTTP dev host with a port), not the // configured publisher domain. let mut settings = test_settings(); - settings.auction.rewrite_creatives = true; settings.publisher.domain = "example.com".to_string(); let mut winning_bids = HashMap::new(); @@ -9709,9 +8234,6 @@ mod tests { height: 250, nurl: None, burl: None, - bid_id: None, - creative_id: None, - renderer: None, ad_id: Some("bid-impression-id".to_string()), cache_id: Some("f47447a0-b759-4f2f-9887-af458b79b570".to_string()), cache_host: Some("openads.adsrvr.org".to_string()), @@ -9764,10 +8286,7 @@ mod tests { height: 250, nurl: None, burl: None, - bid_id: None, ad_id: Some("aps-bid-token".to_string()), - creative_id: None, - renderer: None, cache_id: None, cache_host: None, cache_path: None, @@ -9801,65 +8320,6 @@ mod tests { ); } - #[test] - fn bid_map_exposes_aps_renderer_and_selected_bid_id() { - // Sanitization is opt-in, so enable it: the script-only creative - // below is what drives this bid onto the renderer path. Left at the - // default it would survive processing as an ordinary creative. - let mut settings = test_settings(); - settings.auction.sanitize_creatives = true; - let mut bid = make_bid("atf_sidebar_ad", 1.50, "aps", "fallback-ad", "", ""); - bid.bid_id = Some("selected-bid".to_string()); - bid.creative = Some("".to_string()); - bid.nurl = None; - bid.burl = None; - bid.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "selected-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); - let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); - - let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); - let obj = map["atf_sidebar_ad"] - .as_object() - .expect("should include APS bid"); - - assert_eq!(obj["hb_bidder"], "aps"); - assert_eq!(obj["hb_adid"], "selected-bid"); - assert_eq!(obj["renderer"]["type"], "aps"); - assert_eq!(obj["renderer"]["bidId"], "selected-bid"); - assert!(obj.get("adm").is_none()); - - let script = build_bids_script(&map); - assert!(!script.contains("")); - assert!(script.contains("\\u003C/script\\u003E")); - } - - #[test] - fn bid_map_omits_creative_rejected_by_processing_without_renderer() { - // Sanitization is opt-in, so enable it: script-only markup is what - // makes processing reject this bid's only render source. - let mut settings = test_settings(); - settings.auction.sanitize_creatives = true; - let mut bid = make_bid("atf_sidebar_ad", 1.50, "kargo", "fallback-ad", "", ""); - bid.creative = Some("".to_string()); - let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); - - let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); - - assert!( - !map.contains_key("atf_sidebar_ad"), - "should omit a bid whose only creative was rejected" - ); - } - #[test] fn bid_map_omits_hb_adid_when_both_cache_id_and_ad_id_absent() { let mut winning_bids = HashMap::new(); @@ -9876,9 +8336,6 @@ mod tests { height: 250, nurl: None, burl: None, - bid_id: None, - creative_id: None, - renderer: None, ad_id: None, cache_id: None, cache_host: None, @@ -9920,9 +8377,6 @@ mod tests { height: 250, nurl: None, burl: None, - bid_id: None, - creative_id: None, - renderer: None, ad_id: None, cache_id: None, cache_host: None, @@ -9956,72 +8410,24 @@ mod tests { } #[test] - fn bids_script_schedules_ad_init_without_retry_timer() { - let mut map = serde_json::Map::new(); - map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); - - let script = build_bids_script(&map); - - assert!( - script.contains("t.scheduleInitialAdInit"), - "should hand off bids to the deferred adInit scheduler" - ); - assert!( - !script.contains("setTimeout"), - "should not retry adInit on a timer" - ); - assert!( - !script.contains("prevGptSlots"), - "should not use TS-owned slots as adInit success signal" - ); - } - - #[test] - fn bids_script_defers_ad_init_until_after_hydration() { + fn bids_script_calls_ad_init_without_retry_timer() { let mut map = serde_json::Map::new(); map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); - - let script = build_bids_script(&map); - - // adInit() mutates ad-slot subtrees (GPT defineSlot on the - // `-container` wrapper). Running it synchronously at body-parse time - // lands those mutations inside React's hydration window and trips a - // #418 hydration mismatch. The deferral lifecycle (window `load`, - // double `requestAnimationFrame`, generation-0 pinning via - // `tsjs.navGeneration`) lives in the GPT bundle module (with a - // head-injected fallback in gpt_bootstrap.js) where it is executable - // under Vitest (schedule_initial_ad_init.test.ts); this inline - // script must only delegate to that scheduler. - assert!( - script.contains("var s=t.scheduleInitialAdInit"), - "should delegate deferral to the installed scheduler" - ); - // The bids payload is handed to the scheduler (which applies it only - // while the page is still on navigation generation 0) instead of - // being assigned unconditionally, so a faster SPA navigation's live - // bids cannot be clobbered by the stale SSR payload. - assert!( - script.contains("if(typeof s===\"function\")s(b)"), - "should pass the SSR bids payload to the scheduler" - ); - assert!( - script.contains("else t.bids=b"), - "should fall back to a plain bids assignment without a scheduler" - ); - assert!( - !script.contains(".bids=JSON.parse"), - "should not assign the SSR payload unconditionally" - ); - // The one hydration-unsafe thing this script could do is invoke - // adInit synchronously at body-parse time — it must not. + + let script = build_bids_script(&map); + assert!( - !script.contains("adInit()"), - "should not invoke adInit synchronously at parse time" + script.contains("window.tsjs.adInit"), + "should hand off bids to adInit" ); assert!( !script.contains("setTimeout"), "should not retry adInit on a timer" ); + assert!( + !script.contains("prevGptSlots"), + "should not use TS-owned slots as adInit success signal" + ); } #[test] @@ -10091,58 +8497,12 @@ mod tests { ); assert_eq!( request.publisher.page_url.as_deref(), - Some("https://www.example.com/2024/01/my-article/"), - "page_url should use configured publisher identity without client query data" - ); - assert_eq!( - site.page, "https://www.example.com/2024/01/my-article/", - "site.page should use configured publisher identity without client query data" - ); - } - - #[test] - fn auction_request_preserves_configured_publisher_domain_with_query() { - // On the SSAT proxy path the browser addresses the trusted-server - // edge host, but the auction must advertise the configured - // publisher domain to SSPs — otherwise injected creatives and the - // brand-safety pixel leak the edge/staging host. - let slot = make_slot(); - let slots = [slot]; - let slots_ctx = MatchedSlotsContext { - matched_slots: &slots, - request_path: "/2024/01/my-article/?edition=fictional", - }; - let request_info = RequestInfo { - host: "ts.example.com".to_string(), - scheme: "https".to_string(), - }; - - let request = build_auction_request( - &slots_ctx, - None, - &ConsentContext::default(), - &request_info, - "www.example.com", - Some("Mozilla/5.0"), - ); - - assert_eq!( - request.publisher.domain, "www.example.com", - "publisher.domain should be the configured publisher domain, not the edge host" - ); - let site = request.site.expect("should populate site metadata"); - assert_eq!( - site.domain, "www.example.com", - "site.domain should be the configured publisher domain, not the edge host" - ); - assert_eq!( - request.publisher.page_url.as_deref(), - Some("https://www.example.com/2024/01/my-article/"), - "page_url should remove client query data" + Some("https://www.example.com/2024/01/my-article/?edition=fictional"), + "page_url host should be the configured publisher domain, not the edge host" ); assert_eq!( - site.page, "https://www.example.com/2024/01/my-article/", - "site.page should remove client query data" + site.page, "https://www.example.com/2024/01/my-article/?edition=fictional", + "site.page host should be the configured publisher domain, not the edge host" ); } @@ -10226,108 +8586,11 @@ mod tests { mod page_bids_no_match_tests { use super::super::*; - use super::build_services_with_http_client; use crate::auction::AuctionOrchestrator; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; - use crate::auction::types::{AuctionRequest, AuctionResponse, Bid}; use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; - use crate::platform::test_support::{StubHttpClient, noop_services}; - use crate::platform::{PlatformHttpRequest, PlatformResponse}; + use crate::platform::test_support::noop_services; use crate::test_support::tests::crate_test_settings_str; - use error_stack::{Report, ResultExt}; use http::Method; - use std::sync::{Arc, Mutex}; - - const AUCTION_ID_TEST_PROVIDER: &str = "auction_id_test_provider"; - const AUCTION_ID_TEST_BACKEND: &str = "auction-id-test-backend"; - - struct AuctionIdTestProvider { - captured_request: Arc>>, - winning_bid: bool, - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for AuctionIdTestProvider { - fn provider_name(&self) -> &'static str { - AUCTION_ID_TEST_PROVIDER - } - - async fn request_bids( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - *self - .captured_request - .lock() - .expect("should lock captured auction request") = Some(request.clone()); - let request = PlatformHttpRequest::new( - Request::builder() - .method(Method::POST) - .uri("https://bidder.example.test/bids") - .body(EdgeBody::empty()) - .expect("should build test bidder request"), - AUCTION_ID_TEST_BACKEND, - ); - context - .services - .http_client() - .send_async(request) - .await - .change_context(TrustedServerError::Auction { - message: "test bidder launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - let bids = if self.winning_bid { - vec![Bid { - slot_id: "atf".to_string(), - price: Some(1.50), - currency: "USD".to_string(), - creative: None, - adomain: None, - bidder: AUCTION_ID_TEST_PROVIDER.to_string(), - width: 300, - height: 250, - nurl: None, - burl: None, - bid_id: None, - creative_id: None, - renderer: None, - ad_id: Some("winner-123".to_string()), - cache_id: None, - cache_host: None, - cache_path: None, - metadata: Default::default(), - }] - } else { - Vec::new() - }; - Ok(AuctionResponse::success( - AUCTION_ID_TEST_PROVIDER, - bids, - response_time_ms, - )) - } - - fn timeout_ms(&self) -> u32 { - 100 - } - - fn backend_name( - &self, - _services: &RuntimeServices, - _timeout_ms: u32, - ) -> Option { - Some(AUCTION_ID_TEST_BACKEND.to_string()) - } - } fn settings_with_co() -> Settings { let toml = format!( @@ -10408,7 +8671,6 @@ mod tests { targeting: Default::default(), providers: Default::default(), compiled_patterns: Vec::new(), - compiled_unit: None, }] } @@ -10472,206 +8734,6 @@ mod tests { .expect("should return ok response") } - fn auction_id_test_orchestrator( - settings: &Settings, - captured_request: Arc>>, - winning_bid: bool, - ) -> AuctionOrchestrator { - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(AuctionIdTestProvider { - captured_request, - winning_bid, - })); - orchestrator - } - - #[tokio::test] - async fn page_bids_response_includes_auction_id_only_for_winning_bids() { - let mut settings = settings_with_co(); - settings.auction.providers = vec![AUCTION_ID_TEST_PROVIDER.to_string()]; - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); - let slots = article_slot(); - let winning_stub = Arc::new(StubHttpClient::new()); - winning_stub.push_response(200, b"winner".to_vec()); - let winning_services = build_services_with_http_client( - Arc::clone(&winning_stub) as Arc - ); - let winning_request = Arc::new(Mutex::new(None)); - let winning_orchestrator = - auction_id_test_orchestrator(&settings, Arc::clone(&winning_request), true); - let ec_context = EcContext::new_for_test( - Some("page-auction-example-123".to_string()), - crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }, - ); - - let winning_response = handle_page_bids( - &settings, - &winning_services, - None, - AuctionDispatch { - orchestrator: &winning_orchestrator, - slots: &slots, - registry: None, - }, - &ec_context, - make_page_bids_request("/2024/01/my-article/"), - ) - .await - .expect("should return winning page-bids response"); - let winning_body: serde_json::Value = serde_json::from_slice( - &winning_response - .into_body() - .into_bytes() - .expect("should read winning page-bids response body"), - ) - .expect("should serialize winning page-bids response as JSON"); - let auction_request = winning_request - .lock() - .expect("should lock captured winning request") - .clone() - .expect("should dispatch a winning auction request"); - - assert_eq!( - auction_request.id, "ts-page-auction-example-123", - "test EC ID should produce a deterministic auction request ID" - ); - let winning_auction_id = winning_body["bids"]["atf"]["hb_auction_id"] - .as_str() - .expect("page-bids should expose an auction ID on the winner") - .to_string(); - assert!( - winning_auction_id.starts_with("ts-auc-"), - "page-bids should expose a freshly minted diagnostics token, got `{winning_auction_id}`" - ); - assert_ne!( - winning_auction_id, auction_request.id, - "browser-visible auction ID must not be the EC-derived request ID" - ); - assert!( - !winning_auction_id.contains("page-auction-example-123"), - "browser-visible auction ID must not embed the EC ID" - ); - - let no_winner_stub = Arc::new(StubHttpClient::new()); - no_winner_stub.push_response(200, b"no-bid".to_vec()); - let no_winner_services = build_services_with_http_client( - Arc::clone(&no_winner_stub) as Arc - ); - let no_winner_orchestrator = - auction_id_test_orchestrator(&settings, Arc::new(Mutex::new(None)), false); - let no_winner_response = handle_page_bids( - &settings, - &no_winner_services, - None, - AuctionDispatch { - orchestrator: &no_winner_orchestrator, - slots: &slots, - registry: None, - }, - &ec_context, - make_page_bids_request("/2024/01/my-article/"), - ) - .await - .expect("should return no-winner page-bids response"); - let no_winner_body: serde_json::Value = serde_json::from_slice( - &no_winner_response - .into_body() - .into_bytes() - .expect("should read no-winner page-bids response body"), - ) - .expect("should serialize no-winner page-bids response as JSON"); - - assert!( - no_winner_body["bids"] - .as_object() - .expect("page-bids should return a bids object") - .is_empty(), - "page-bids should not fabricate auction metadata without a winner" - ); - } - - /// The browser-visible auction ID is minted per auction and only for - /// deployments that run the diagnostics integration, so it can neither - /// carry EC identity across auctions nor reach pages that ignore it. - #[tokio::test] - async fn page_bids_auction_id_is_per_auction_and_gated_on_diagnostics() { - async fn winning_auction_id(settings: &Settings) -> Option { - let slots = article_slot(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"winner".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let orchestrator = - auction_id_test_orchestrator(settings, Arc::new(Mutex::new(None)), true); - let ec_context = EcContext::new_for_test( - Some("page-auction-example-123".to_string()), - crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }, - ); - let response = handle_page_bids( - settings, - &services, - None, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &slots, - registry: None, - }, - &ec_context, - make_page_bids_request("/2024/01/my-article/"), - ) - .await - .expect("should return page-bids response"); - let body: serde_json::Value = serde_json::from_slice( - &response - .into_body() - .into_bytes() - .expect("should read page-bids response body"), - ) - .expect("should serialize page-bids response as JSON"); - body["bids"]["atf"]["hb_auction_id"] - .as_str() - .map(str::to_string) - } - - let mut settings = settings_with_co(); - settings.auction.providers = vec![AUCTION_ID_TEST_PROVIDER.to_string()]; - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); - - let first = winning_auction_id(&settings) - .await - .expect("first auction should expose a diagnostics token"); - let second = winning_auction_id(&settings) - .await - .expect("second auction should expose a diagnostics token"); - assert_ne!( - first, second, - "each auction for the same visitor should mint its own token" - ); - - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": false })) - .expect("should disable diagnostics"); - assert_eq!( - winning_auction_id(&settings).await, - None, - "no auction metadata should reach the page without the diagnostics integration" - ); - } - /// The deprecated `/__ts/page-bids` alias must be handled identically to /// the canonical path — same status, same JSON body. /// @@ -10972,46 +9034,6 @@ mod tests { ); } - #[tokio::test] - async fn page_bids_omits_only_over_limit_dynamic_slot() { - let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let mut over_limit = article_slot() - .into_iter() - .next() - .expect("should build over-limit slot"); - over_limit.id = "over_limit_dynamic".to_string(); - over_limit.page_patterns = vec!["/*".to_string()]; - over_limit.gam_unit_path = Some("/{section}/{section}".to_string()); - over_limit - .compile_unit_template() - .expect("template should compile"); - let mut valid_static = article_slot() - .into_iter() - .next() - .expect("should build valid static slot"); - valid_static.id = "valid_static_sibling".to_string(); - valid_static.page_patterns = vec!["/*".to_string()]; - valid_static.gam_unit_path = Some("/12345/example/static".to_string()); - let slots = vec![over_limit, valid_static]; - let request_path = format!("/{}", "a".repeat(60)); - let mut req = make_page_bids_request(&request_path); - set_test_header(&mut req, "sec-purpose", "prefetch"); - - let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; - let returned_slots = body["slots"].as_array().expect("slots should be array"); - - assert_eq!( - returned_slots.len(), - 1, - "should omit only the over-limit dynamic slot" - ); - assert_eq!( - returned_slots[0]["id"], "valid_static_sibling", - "should retain the valid static sibling" - ); - } - #[tokio::test] async fn url_not_matching_any_pattern_returns_empty_response() { // Slots exist but request path does not match — no auction, no injection. @@ -11181,7 +9203,7 @@ mod tests { /// the handler emitted. mod navigation_publisher_domain_tests { use super::*; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::provider::AuctionProvider; use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; use crate::auction::types::AuctionRequest; use crate::auction::{AuctionContext, AuctionOrchestrator}; @@ -11189,7 +9211,7 @@ mod tests { use crate::platform::test_support::{ NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, }; - use crate::platform::{ClientInfo, PlatformResponse}; + use crate::platform::{ClientInfo, PlatformPendingRequest, PlatformResponse}; use crate::test_support::tests::crate_test_settings_str; use std::sync::Mutex; @@ -11218,7 +9240,7 @@ mod tests { &self, request: &AuctionRequest, _context: &AuctionContext<'_>, - ) -> Result> { + ) -> Result> { *self.captured.lock().expect("should lock captured request") = Some(request.clone()); Err(Report::new(TrustedServerError::Auction { @@ -11291,42 +9313,9 @@ mod tests { targeting: Default::default(), providers: Default::default(), compiled_patterns: Vec::new(), - compiled_unit: None, }] } - fn slots_with_over_limit_dynamic_sibling() -> Vec { - let mut over_limit = article_slot() - .into_iter() - .next() - .expect("should build over-limit slot"); - over_limit.id = "over_limit_dynamic".to_string(); - over_limit.page_patterns = vec!["/*".to_string()]; - over_limit.gam_unit_path = Some("/{section}/{section}".to_string()); - - let mut valid_static = article_slot() - .into_iter() - .next() - .expect("should build valid static slot"); - valid_static.id = "valid_static_sibling".to_string(); - valid_static.page_patterns = vec!["/*".to_string()]; - valid_static.gam_unit_path = Some("/12345/example/static".to_string()); - - vec![over_limit, valid_static] - } - - fn assert_only_renderable_slot_was_auctioned( - captured: &Arc>>, - ) { - let request = captured - .lock() - .expect("should lock captured request") - .clone() - .expect("should dispatch an auction request"); - let slot_ids: Vec<_> = request.slots.iter().map(|slot| slot.id.as_str()).collect(); - assert_eq!(slot_ids, vec!["valid_static_sibling"]); - } - /// [`EcContext`] whose consent context permits the server-side auction. fn consent_allowing_ec_context() -> EcContext { let consent = crate::consent::ConsentContext { @@ -11501,90 +9490,5 @@ mod tests { assert_configured_domain(&captured, &telemetry_sink); } - - #[tokio::test] - async fn initial_navigation_auctions_only_renderable_slots() { - let settings = settings_with_capturing_provider(); - let captured = Arc::new(Mutex::new(None)); - let orchestrator = orchestrator_capturing_request(&settings, &captured); - let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = services_with( - Arc::clone(&stub) as Arc, - telemetry_sink, - ); - let mut ec_context = consent_allowing_ec_context(); - let request_path = format!("/{}", "a".repeat(60)); - let req = HttpRequest::builder() - .method(Method::GET) - .uri(format!("https://{EDGE_HOST}{request_path}")) - .header(header::HOST, EDGE_HOST) - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build test request"); - let slots = slots_with_over_limit_dynamic_sibling(); - - let _ = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &slots, - registry: None, - }, - req, - ) - .await - .expect("should proxy publisher request"); - - assert_only_renderable_slot_was_auctioned(&captured); - } - - #[tokio::test] - async fn page_bids_auctions_only_renderable_slots() { - let settings = settings_with_capturing_provider(); - let captured = Arc::new(Mutex::new(None)); - let orchestrator = orchestrator_capturing_request(&settings, &captured); - let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let services = services_with( - Arc::new(crate::platform::test_support::NoopHttpClient), - telemetry_sink, - ); - let ec_context = consent_allowing_ec_context(); - let request_path = format!("/{}", "a".repeat(60)); - let mut req = HttpRequest::builder() - .method(Method::GET) - .uri(format!( - "https://{EDGE_HOST}/_ts/page-bids?path={request_path}" - )) - .header(header::HOST, EDGE_HOST) - .body(EdgeBody::empty()) - .expect("should build test request"); - req.headers_mut().insert( - header::HeaderName::from_static("sec-fetch-site"), - HeaderValue::from_static("same-origin"), - ); - let slots = slots_with_over_limit_dynamic_sibling(); - - let _ = handle_page_bids( - &settings, - &services, - None, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &slots, - registry: None, - }, - &ec_context, - req, - ) - .await - .expect("should return ok response"); - - assert_only_renderable_slot_was_auctioned(&captured); - } } } diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 21ba9f20b..40650d7e7 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -11,38 +11,31 @@ use edgezero_core::http::{HeaderName, HeaderValue, Response, header}; +use crate::cache_policy::{ + CacheControlPolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, + is_edge_cache_header_name, remove_edge_cache_headers, +}; use crate::settings::Settings; -/// CDN-targeted cache headers stripped from every cookie-bearing response. -/// -/// A single source of truth so the adapter copies of the privacy downgrade -/// cannot drift apart. -pub const CDN_CACHE_HEADERS: &[&str] = &[ - "surrogate-control", - "fastly-surrogate-control", - "cdn-cache-control", - "cloudflare-cdn-cache-control", -]; - -fn strip_cdn_cache_headers(response: &mut Response) { - for name in CDN_CACHE_HEADERS { - response.headers_mut().remove(*name); - } +/// Runtime edge-cache headers stripped from private or cookie-bearing responses. +pub use crate::cache_policy::EDGE_CACHE_HEADER_NAMES as SURROGATE_CACHE_HEADERS; +/// Backwards-compatible name used by integrations that clear every edge-cache directive. +pub use crate::cache_policy::EDGE_CACHE_HEADER_NAMES as CDN_CACHE_HEADERS; + +fn cache_control_is_private_or_no_store(response: &Response) -> bool { + cache_control_headers_are_private_or_no_store(response.headers()) } /// Forces synthesized HTML to be private and non-storable. /// /// Use this exact policy whenever Trusted Server changes an origin HTML /// representation with request-specific content: force `private, no-store`, -/// remove origin validators, and remove all CDN-targeted cache directives. +/// remove origin validators, and remove all runtime edge-cache directives. pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); + CacheControlPolicy::NoStorePrivate + .apply_to_headers(response.headers_mut(), EdgeCacheHeader::None); response.headers_mut().remove(header::ETAG); response.headers_mut().remove(header::LAST_MODIFIED); - strip_cdn_cache_headers(response); } /// Forces cookie-bearing responses to stay private to shared caches. @@ -58,19 +51,14 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { if !response.headers().contains_key(header::SET_COOKIE) { return; } - // Shared-cache control headers must come off every cookie-bearing response, even - // one already carrying a stricter `no-store`/`private` directive — they are + // Edge-cache headers must come off every cookie-bearing response, even one + // already carrying a stricter `no-store`/`private` directive — they are // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. - strip_cdn_cache_headers(response); + remove_edge_cache_headers(response.headers_mut()); // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match // against a lowercased copy — `No-Store` / `Private` must count. - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let already_uncacheable = cache_control_is_private_or_no_store(response); if !already_uncacheable { response.headers_mut().insert( header::CACHE_CONTROL, @@ -85,10 +73,10 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { /// First downgrades cookie-bearing responses via /// [`enforce_set_cookie_cache_privacy`], then applies operator headers — but on /// an uncacheable (`private`/`no-store`) response the cache-controlling headers -/// (`Cache-Control` and the surrogate cache headers) are skipped so operators +/// (`Cache-Control` and runtime edge-cache headers) are skipped so operators /// cannot re-enable shared caching for per-user payloads. After the operator /// headers are applied the cookie-privacy downgrade runs once more, so a -/// configured `Set-Cookie` combined with public/surrogate cache headers cannot +/// configured `Set-Cookie` combined with public edge-cache headers cannot /// produce a shared-cacheable cookie-bearing response. /// /// Invalid header names/values are logged and skipped rather than panicking, so @@ -96,19 +84,15 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: &mut Response) { enforce_set_cookie_cache_privacy(response); - let response_is_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let response_is_uncacheable = cache_control_is_private_or_no_store(response); + if response_is_uncacheable { + remove_edge_cache_headers(response.headers_mut()); + } for (key, value) in &settings.response_headers { if response_is_uncacheable && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) - || CDN_CACHE_HEADERS - .iter() - .any(|name| key.eq_ignore_ascii_case(name))) + || is_edge_cache_header_name(key)) { continue; } @@ -129,10 +113,14 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: response.headers_mut().insert(header_name, header_value); } + if cache_control_is_private_or_no_store(response) { + remove_edge_cache_headers(response.headers_mut()); + } + // Operator headers can themselves introduce Set-Cookie (alongside public - // or surrogate cache headers) onto a previously cookieless response, which - // the pre-apply pass could not see. Re-run the downgrade so the final - // response can never pair Set-Cookie with shared cacheability. + // edge-cache headers) onto a previously cookieless response, which the + // pre-apply pass could not see. Re-run the downgrade so the final response + // can never pair Set-Cookie with shared cacheability. enforce_set_cookie_cache_privacy(response); } @@ -169,7 +157,7 @@ mod tests { } #[test] - fn synthesized_html_is_forced_no_store_without_validators_or_cdn_headers() { + fn synthesized_html_is_forced_no_store_without_validators_or_edge_headers() { let mut response = response_builder() .header(header::CACHE_CONTROL, "private, max-age=600") .header(header::ETAG, "\"origin\"") @@ -185,12 +173,12 @@ mod tests { assert_eq!( response.headers()[header::CACHE_CONTROL], - "private, no-store", + "no-store, private", "synthesized HTML should always be non-storable" ); for header_name in [header::ETAG.as_str(), header::LAST_MODIFIED.as_str()] .into_iter() - .chain(CDN_CACHE_HEADERS.iter().copied()) + .chain(SURROGATE_CACHE_HEADERS.iter().copied()) { assert!( !response.headers().contains_key(header_name), @@ -205,7 +193,6 @@ mod tests { let mut response = response_builder() .header(header::SET_COOKIE, "id=abc") .header("surrogate-control", "max-age=600") - .header("fastly-surrogate-control", "max-age=600") .header("cdn-cache-control", "max-age=600") .header("cloudflare-cdn-cache-control", "max-age=600") .body(edgezero_core::body::Body::empty()) @@ -221,12 +208,14 @@ mod tests { Some("private, max-age=0"), "operator public Cache-Control must not override cookie privacy downgrade" ); - for header_name in CDN_CACHE_HEADERS { - assert!( - !response.headers().contains_key(*header_name), - "CDN cache header {header_name} must be stripped on cookie responses" - ); - } + assert!( + !response.headers().contains_key("surrogate-control") + && !response.headers().contains_key("cdn-cache-control") + && !response + .headers() + .contains_key("cloudflare-cdn-cache-control"), + "edge cache headers must be stripped on cookie responses" + ); } #[test] @@ -237,9 +226,8 @@ mod tests { ("set-cookie", "operator=abc"), ("cache-control", "public, max-age=600"), ("surrogate-control", "max-age=600"), - ("fastly-surrogate-control", "max-age=600"), - ("cdn-cache-control", "public, max-age=600"), - ("cloudflare-cdn-cache-control", "public, max-age=600"), + ("cdn-cache-control", "max-age=600"), + ("cloudflare-cdn-cache-control", "max-age=600"), ]); let mut response = response_builder() .body(edgezero_core::body::Body::empty()) @@ -255,18 +243,49 @@ mod tests { Some("private, max-age=0"), "operator Set-Cookie plus public Cache-Control must be re-downgraded to private" ); - for header_name in CDN_CACHE_HEADERS { - assert!( - !response.headers().contains_key(*header_name), - "CDN cache header {header_name} must be stripped when operator headers add Set-Cookie" - ); - } + assert!( + !response.headers().contains_key("surrogate-control") + && !response.headers().contains_key("cdn-cache-control") + && !response + .headers() + .contains_key("cloudflare-cdn-cache-control"), + "edge cache headers must be stripped when operator headers add Set-Cookie" + ); assert!( response.headers().contains_key(header::SET_COOKIE), "the operator Set-Cookie itself should still be applied" ); } + #[test] + fn cookie_privacy_does_not_treat_pseudo_directives_as_uncacheable() { + let settings = settings_with_response_headers(&[]); + let mut response = response_builder() + .header(header::SET_COOKIE, "id=abc") + .header( + header::CACHE_CONTROL, + "public, max-age=600, no-storey, not-private", + ) + .header("surrogate-control", "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "pseudo-directives must not prevent the cookie privacy downgrade" + ); + assert!( + !response.headers().contains_key("surrogate-control"), + "cookie privacy downgrade should still strip edge-cache headers" + ); + } + #[test] fn preserves_private_no_store_against_operator_cache_headers_without_cookie() { let settings = settings_with_response_headers(&[ @@ -288,7 +307,7 @@ mod tests { "private, no-store", "operator cache headers must not weaken an existing private response" ); - for header_name in CDN_CACHE_HEADERS { + for header_name in SURROGATE_CACHE_HEADERS { assert!( !response.headers().contains_key(*header_name), "operator headers must not restore shared caching through {header_name}" @@ -297,42 +316,47 @@ mod tests { } #[test] - fn applies_operator_headers_on_cookieless_response() { - let settings = settings_with_response_headers(&[("x-operator", "value")]); + fn strips_edge_headers_from_uncacheable_cookieless_response() { + let settings = settings_with_response_headers(&[ + ("cdn-cache-control", "max-age=600"), + ("cloudflare-cdn-cache-control", "max-age=600"), + ]); let mut response = response_builder() + .header(header::CACHE_CONTROL, "private, max-age=0") + .header("surrogate-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") .body(edgezero_core::body::Body::empty()) .expect("should build response"); apply_response_headers_with_cache_privacy(&settings, &mut response); - assert_eq!( - response - .headers() - .get("x-operator") - .and_then(|v| v.to_str().ok()), - Some("value"), - "operator headers should still apply to cacheable responses" + assert!( + !response.headers().contains_key("surrogate-control") + && !response.headers().contains_key("cdn-cache-control") + && !response + .headers() + .contains_key("cloudflare-cdn-cache-control"), + "uncacheable responses must not retain or receive edge-cache headers" ); } #[test] - fn uncacheable_response_rejects_operator_cdn_cache_headers() { - let settings = settings_with_response_headers(&[ - ("cdn-cache-control", "public, max-age=600"), - ("cloudflare-cdn-cache-control", "public, max-age=600"), - ]); + fn applies_operator_headers_on_cookieless_response() { + let settings = settings_with_response_headers(&[("x-operator", "value")]); let mut response = response_builder() - .header(header::CACHE_CONTROL, "private, no-store") .body(edgezero_core::body::Body::empty()) .expect("should build response"); apply_response_headers_with_cache_privacy(&settings, &mut response); - for header_name in ["cdn-cache-control", "cloudflare-cdn-cache-control"] { - assert!( - !response.headers().contains_key(header_name), - "operator headers must not restore shared caching through {header_name}" - ); - } + assert_eq!( + response + .headers() + .get("x-operator") + .and_then(|v| v.to_str().ok()), + Some("value"), + "operator headers should still apply to cacheable responses" + ); } } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 598c00056..ad17595f9 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1,6 +1,7 @@ #[cfg(test)] use config::{Config, Environment, File, FileFormat}; use error_stack::{Report, ResultExt}; +use glob::Pattern; use regex::Regex; use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; @@ -8,10 +9,12 @@ use std::collections::{HashMap, HashSet}; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; +use std::time::Duration; use url::Url; use validator::{Validate, ValidationError}; use crate::auction_config_types::AuctionConfig; +use crate::cache_policy::{CachePolicy, CacheVisibility}; use crate::consent_config::ConsentConfig; use crate::creative_opportunities::CreativeOpportunitiesConfig; use crate::error::TrustedServerError; @@ -1866,6 +1869,322 @@ fn validate_tinybird_secret(value: &str, setting: &str) -> Result<(), Report, +} + +impl CacheSettings { + fn normalize(&mut self) { + for rule in &mut self.asset_rules { + rule.normalize(); + } + } + + /// Eagerly validate runtime-only cache settings artifacts. + /// + /// # Errors + /// + /// Returns a configuration error if any rule ID is duplicate, matcher shape + /// is invalid, or a configured regex/glob cannot compile. + pub fn prepare_runtime(&self) -> Result<(), Report> { + let mut seen_ids = HashSet::new(); + for rule in &self.asset_rules { + if rule.id.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: "cache.asset_rules id must not be empty".to_string(), + })); + } + if !seen_ids.insert(rule.id.clone()) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("cache.asset_rules contains duplicate id `{}`", rule.id), + })); + } + rule.prepare_runtime()?; + } + Ok(()) + } + + /// Resolve the first enabled asset cache rule that matches `path`. + /// + /// # Errors + /// + /// Returns a configuration error if a lazily prepared matcher unexpectedly + /// fails to compile. + pub fn asset_policy_for_path( + &self, + path: &str, + ) -> Result, Report> { + for rule in &self.asset_rules { + if rule.matches_path(path)? { + return Ok(Some(rule.cache_policy())); + } + } + Ok(None) + } +} + +/// A configurable cache rule for publisher-origin or rehosted static assets. +#[derive(Debug, Default, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CacheAssetRule { + /// Stable operator-facing identifier for logs/tests/config errors. + pub id: String, + /// Whether this rule participates in matching. + #[serde(default)] + pub enabled: bool, + /// Built-in framework/static preset matcher. + #[serde(default)] + pub preset: Option, + /// Raw path prefix matcher. + #[serde(default)] + pub path_prefix: Option, + /// Single glob matcher retained for concise configs. + #[serde(default)] + pub path_glob: Option, + /// Multiple glob matchers. + #[serde(default)] + pub path_globs: Vec, + /// Regex matcher applied to the request path. + #[serde(default)] + pub path_regex: Option, + /// File extensions matched against the request path, case-insensitively. + #[serde(default)] + pub extensions: Vec, + /// Require a hash-like token in the final path segment before the rule matches. + #[serde(default)] + pub requires_hash_in_filename: bool, + /// Browser-facing cache visibility. + #[serde(default)] + pub visibility: CachePolicyVisibility, + /// Browser cache TTL rendered as `max-age`. + #[serde(default)] + pub browser_ttl_seconds: Option, + /// Shared edge cache TTL rendered as runtime-specific edge control. + #[serde(default)] + pub edge_ttl_seconds: Option, + /// Optional stale-while-revalidate duration. + #[serde(default)] + pub stale_while_revalidate_seconds: Option, + /// Optional stale-if-error duration. + #[serde(default)] + pub stale_if_error_seconds: Option, + /// Whether browser caches may treat the response as immutable. + #[serde(default)] + pub immutable: bool, + #[serde(skip)] + compiled_regex: OnceLock>, + #[serde(skip)] + compiled_globs: OnceLock, String>>, +} + +impl CacheAssetRule { + fn normalize(&mut self) { + self.id = self.id.trim().to_string(); + self.path_prefix = self + .path_prefix + .take() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + self.path_glob = self + .path_glob + .take() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + self.path_globs = self + .path_globs + .iter() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .collect(); + self.path_regex = self + .path_regex + .take() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + self.extensions = self + .extensions + .iter() + .map(|value| value.trim().trim_start_matches('.').to_ascii_lowercase()) + .filter(|value| !value.is_empty()) + .collect(); + } + + fn prepare_runtime(&self) -> Result<(), Report> { + self.validate_matcher_shape()?; + self.compiled_regex().map(|_| ())?; + self.compiled_globs().map(|_| ())?; + Ok(()) + } + + fn validate_matcher_shape(&self) -> Result<(), Report> { + if self.path_glob.is_some() && !self.path_globs.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` must use path_glob or path_globs, not both", + self.id + ), + })); + } + + let matcher_count = usize::from(self.preset.is_some()) + + usize::from(self.path_prefix.is_some()) + + usize::from(self.path_glob.is_some() || !self.path_globs.is_empty()) + + usize::from(self.path_regex.is_some()) + + usize::from(!self.extensions.is_empty()); + + if matcher_count != 1 { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` must configure exactly one matcher", + self.id + ), + })); + } + Ok(()) + } + + fn compiled_regex(&self) -> Result, Report> { + let Some(pattern) = self.path_regex.as_deref() else { + return Ok(None); + }; + match self + .compiled_regex + .get_or_init(|| Regex::new(pattern).map_err(|err| err.to_string())) + { + Ok(regex) => Ok(Some(regex)), + Err(message) => Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` path_regex `{pattern}` failed to compile: {message}", + self.id + ), + })), + } + } + + fn compiled_globs(&self) -> Result, Report> { + if self.path_glob.is_none() && self.path_globs.is_empty() { + return Ok(None); + } + + match self.compiled_globs.get_or_init(|| { + if let Some(glob) = self.path_glob.as_deref() { + Pattern::new(glob) + .map(|pattern| vec![pattern]) + .map_err(|err| err.to_string()) + } else { + self.path_globs + .iter() + .map(|pattern| Pattern::new(pattern).map_err(|err| err.to_string())) + .collect() + } + }) { + Ok(patterns) => Ok(Some(patterns.as_slice())), + Err(message) => Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` glob matcher failed to compile: {message}", + self.id + ), + })), + } + } + + fn matches_path(&self, path: &str) -> Result> { + if !self.enabled { + return Ok(false); + } + if self.requires_hash_in_filename && !filename_contains_hash(path) { + return Ok(false); + } + + if let Some(preset) = self.preset { + return Ok(preset.matches_path(path)); + } + if let Some(prefix) = self.path_prefix.as_deref() { + return Ok(path.starts_with(prefix)); + } + if let Some(patterns) = self.compiled_globs()? { + return Ok(patterns.iter().any(|pattern| pattern.matches(path))); + } + if let Some(regex) = self.compiled_regex()? { + return Ok(regex.is_match(path)); + } + if !self.extensions.is_empty() { + return Ok(path_extension(path).is_some_and(|extension| { + self.extensions + .iter() + .any(|candidate| candidate == &extension) + })); + } + Ok(false) + } + + fn cache_policy(&self) -> CachePolicy { + CachePolicy { + visibility: self.visibility.into(), + browser_ttl: self.browser_ttl_seconds.map(Duration::from_secs), + edge_ttl: self.edge_ttl_seconds.map(Duration::from_secs), + stale_while_revalidate: self.stale_while_revalidate_seconds.map(Duration::from_secs), + stale_if_error: self.stale_if_error_seconds.map(Duration::from_secs), + immutable: self.immutable, + } + } +} + +/// Built-in cache-rule presets that operators can enable explicitly. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub enum CacheAssetPreset { + /// Next.js build output under `/_next/static/`. + #[serde(rename = "nextjs-static")] + NextJsStatic, +} + +impl CacheAssetPreset { + fn matches_path(self, path: &str) -> bool { + match self { + Self::NextJsStatic => path.starts_with("/_next/static/"), + } + } +} + +/// Cache visibility parsed from operator configuration. +#[derive(Debug, Default, Clone, Copy, Deserialize, Serialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub enum CachePolicyVisibility { + /// Public browser/cache visibility. + #[default] + Public, + /// Private browser visibility. + Private, +} + +impl From for CacheVisibility { + fn from(value: CachePolicyVisibility) -> Self { + match value { + CachePolicyVisibility::Public => Self::Public, + CachePolicyVisibility::Private => Self::Private, + } + } +} + +fn path_extension(path: &str) -> Option { + let filename = path.rsplit('/').next()?; + let (_, extension) = filename.rsplit_once('.')?; + (!extension.is_empty()).then(|| extension.to_ascii_lowercase()) +} + +fn filename_contains_hash(path: &str) -> bool { + let filename = path.rsplit('/').next().unwrap_or(path); + filename + .split(['.', '-', '_', '~']) + .any(|segment| segment.len() >= 8 && segment.chars().all(|ch| ch.is_ascii_hexdigit())) +} + /// Debug-only features. All flags default to `false` (off in production). #[derive(Debug, Default, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -1937,6 +2256,9 @@ pub struct Settings { #[serde(default)] pub consent: ConsentConfig, #[serde(default)] + #[validate(nested)] + pub cache: CacheSettings, + #[serde(default)] pub proxy: Proxy, #[serde(default)] pub creative_opportunities: Option, @@ -2019,6 +2341,8 @@ impl Settings { mut settings: Self, validation_label: &str, ) -> Result> { + settings.integrations.normalize(); + settings.cache.normalize(); settings.proxy.normalize(); settings.image_optimizer.normalize(); settings.consent.validate(); @@ -2052,6 +2376,7 @@ impl Settings { /// opportunity slot is invalid. pub fn prepare_runtime(&mut self) -> Result<(), Report> { self.image_optimizer.prepare_runtime()?; + self.cache.prepare_runtime()?; self.proxy.prepare_runtime()?; self.tinybird.prepare_runtime()?; self.validate_asset_image_optimizer_profile_sets()?; @@ -2167,6 +2492,18 @@ impl Settings { Ok(()) } + /// Resolve the first matching configured asset cache policy for the request path. + /// + /// # Errors + /// + /// Returns a configuration error if matcher preparation unexpectedly fails. + pub fn asset_cache_policy_for_path( + &self, + path: &str, + ) -> Result, Report> { + self.cache.asset_policy_for_path(path) + } + /// Resolve the longest matching asset route for the request path. #[must_use] pub fn asset_route_for_path(&self, path: &str) -> Option<&ProxyAssetRoute> { @@ -2729,6 +3066,143 @@ mod tests { ); } + #[test] + fn cache_asset_rule_nextjs_preset_is_operator_controlled() { + let toml_str = format!( + r#"{} + + [[cache.asset_rules]] + id = "nextjs-static" + enabled = true + preset = "nextjs-static" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse cache asset rule"); + + let policy = settings + .asset_cache_policy_for_path("/_next/static/chunks/app.js") + .expect("should evaluate cache rules") + .expect("should match enabled Next.js preset"); + assert_eq!( + policy, + CachePolicy::public_immutable(Duration::from_secs(31_536_000)), + "enabled preset should produce immutable static policy" + ); + + let disabled_toml = toml_str.replace("enabled = true", "enabled = false"); + let disabled_settings = + Settings::from_toml(&disabled_toml).expect("should parse disabled cache asset rule"); + assert!( + disabled_settings + .asset_cache_policy_for_path("/_next/static/chunks/app.js") + .expect("should evaluate disabled cache rules") + .is_none(), + "disabled preset must not mark framework paths immutable" + ); + } + + #[test] + fn cache_asset_rule_requires_hash_in_filename_when_configured() { + let toml_str = format!( + r#"{} + + [[cache.asset_rules]] + id = "publisher-assets" + enabled = true + path_globs = ["/assets/**/*.js"] + requires_hash_in_filename = true + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse cache asset rule"); + + assert!( + settings + .asset_cache_policy_for_path("/assets/app.js") + .expect("should evaluate cache rules") + .is_none(), + "broad allowlist should not match non-fingerprinted files when hash is required" + ); + assert_eq!( + settings + .asset_cache_policy_for_path("/assets/app.0123abcd.js") + .expect("should evaluate cache rules"), + Some(CachePolicy::public_immutable(Duration::from_secs( + 31_536_000 + ))), + "fingerprinted asset should match the allowlist" + ); + } + + #[test] + fn cache_asset_rule_validation_rejects_invalid_config() { + let duplicate_ids = format!( + r#"{} + + [[cache.asset_rules]] + id = "duplicate" + enabled = true + path_prefix = "/assets/" + + [[cache.asset_rules]] + id = "duplicate" + enabled = true + path_prefix = "/static/" + "#, + crate_test_settings_str() + ); + let duplicate_err = + Settings::from_toml(&duplicate_ids).expect_err("should reject duplicate rule ids"); + assert!( + format!("{duplicate_err:?}").contains("duplicate id"), + "should explain duplicate rule id: {duplicate_err:?}" + ); + + let invalid_regex = format!( + r#"{} + + [[cache.asset_rules]] + id = "bad-regex" + enabled = true + path_regex = "[" + "#, + crate_test_settings_str() + ); + let regex_err = + Settings::from_toml(&invalid_regex).expect_err("should reject invalid regex"); + assert!( + format!("{regex_err:?}").contains("path_regex"), + "should explain invalid regex: {regex_err:?}" + ); + + let invalid_shape = format!( + r#"{} + + [[cache.asset_rules]] + id = "too-many-matchers" + enabled = true + path_prefix = "/assets/" + extensions = ["js"] + "#, + crate_test_settings_str() + ); + let shape_err = + Settings::from_toml(&invalid_shape).expect_err("should reject invalid matcher shape"); + assert!( + format!("{shape_err:?}").contains("exactly one matcher"), + "should explain invalid matcher shape: {shape_err:?}" + ); + } + #[test] fn validate_rejects_trailing_slash_in_origin_url() { let toml_str = crate_test_settings_str().replace( diff --git a/crates/trusted-server-core/src/tsjs.rs b/crates/trusted-server-core/src/tsjs.rs index 133e6d011..9f19c62fd 100644 --- a/crates/trusted-server-core/src/tsjs.rs +++ b/crates/trusted-server-core/src/tsjs.rs @@ -1,4 +1,4 @@ -use trusted_server_js::{all_module_ids, concatenated_hash, single_module_hash}; +use trusted_server_js::{concatenated_hash, single_module_hash}; /// `/static` URL for the tsjs bundle with cache-busting hash based on /// the concatenated content of the given module set. @@ -17,25 +17,28 @@ pub fn tsjs_script_tag(module_ids: &[&str]) -> String { ) } -/// `/static` URL for the unified bundle with a conservative cache-busting hash. +/// `/static` URL for the unified bundle when exact module IDs are unavailable. /// -/// Hashes all compiled module IDs so the cache invalidates whenever any module -/// changes. Over-invalidates slightly (includes deferred modules in the hash) -/// but never serves stale content. Use [`tsjs_script_src`] with exact module -/// IDs when `IntegrationRegistry` is available. +/// This intentionally omits `?v=` because the serving path can only mark a URL +/// immutable when the hash matches the exact enabled module set. Use +/// [`tsjs_script_src`] with exact module IDs when [`IntegrationRegistry`] is +/// available. +/// +/// [`IntegrationRegistry`]: crate::integrations::IntegrationRegistry #[must_use] pub fn tsjs_unified_script_src() -> String { - let ids = all_module_ids(); - tsjs_script_src(&ids) + "/static/tsjs=tsjs-unified.min.js".to_string() } -/// `", + tsjs_unified_script_src() + ) } /// `/static` URL for one module with its own cache-busting hash. @@ -171,18 +174,17 @@ mod tests { } #[test] - fn tsjs_unified_helpers_use_all_module_ids() { - let ids = all_module_ids(); + fn tsjs_unified_helpers_use_unversioned_fallback_without_registry() { + let src = tsjs_unified_script_src(); assert_eq!( - tsjs_unified_script_src(), - tsjs_script_src(&ids), - "should hash all module IDs for the unified script source" + src, "/static/tsjs=tsjs-unified.min.js", + "registry-free unified helper should not emit an unverifiable hash" ); assert_eq!( tsjs_unified_script_tag(), - tsjs_script_tag(&ids), - "should wrap the all-module unified script source" + format!(r#""#), + "should wrap the registry-free unified source" ); } @@ -246,14 +248,13 @@ mod tests { } #[test] - fn tsjs_unified_script_src_and_tag_include_cache_busting_hash() { + fn tsjs_unified_script_src_and_tag_omit_unverifiable_cache_busting_hash() { let src = tsjs_unified_script_src(); - assert!( - src.starts_with("/static/tsjs=tsjs-unified.min.js?v="), - "should include unified script URL prefix" + assert_eq!( + src, "/static/tsjs=tsjs-unified.min.js", + "should use the unified script URL without an unverifiable hash" ); - assert_sha256_hex_hash(hash_query_value(&src)); assert_eq!( tsjs_unified_script_tag(), format!(r#""#), diff --git a/crates/trusted-server-js/Cargo.toml b/crates/trusted-server-js/Cargo.toml index f3af9bfcf..67a4ac698 100644 --- a/crates/trusted-server-js/Cargo.toml +++ b/crates/trusted-server-js/Cargo.toml @@ -15,10 +15,11 @@ workspace = true doctest = false name = "trusted_server_js" path = "src/lib.rs" -test = false [build-dependencies] build-print = { workspace = true } +hex = { workspace = true } +sha2 = { workspace = true } which = { workspace = true } [dependencies] diff --git a/crates/trusted-server-js/build.rs b/crates/trusted-server-js/build.rs index 6d6bdde9f..ba6cd88f2 100644 --- a/crates/trusted-server-js/build.rs +++ b/crates/trusted-server-js/build.rs @@ -12,6 +12,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, ExitStatus}; use build_print::{info, warn}; +use sha2::{Digest as _, Sha256}; fn main() { // Rebuild if TS sources change (belt-and-suspenders): enumerate every file under lib/ @@ -125,7 +126,7 @@ fn main() { // Copy each module file to OUT_DIR for (_, filename) in &modules { - copy_bundle(filename, true, &crate_dir, &dist_dir, &out_dir); + copy_bundle(filename, true, &dist_dir, &out_dir); } // Generate tsjs_modules.rs with include_str!() for each module @@ -139,9 +140,10 @@ fn main() { ) .expect("should write generated module header"); for (id, filename) in &modules { + let sha256 = bundle_sha256(&out_dir.join(filename)); writeln!( codegen, - " TsjsModuleMeta {{\n bundle: include_str!(concat!(env!(\"OUT_DIR\"), \"/{filename}\")),\n id: \"{id}\",\n }},\n" + " TsjsModuleMeta {{\n bundle: include_str!(concat!(env!(\"OUT_DIR\"), \"/{filename}\")),\n id: \"{id}\",\n sha256: \"{sha256}\",\n }},\n" ) .expect("should write generated module entry"); } @@ -149,6 +151,7 @@ fn main() { codegen.push_str("\npub(crate) struct TsjsModuleMeta {\n"); codegen.push_str(" pub bundle: &'static str,\n"); codegen.push_str(" pub id: &'static str,\n"); + codegen.push_str(" pub sha256: &'static str,\n"); codegen.push_str("}\n"); let generated_path = out_dir.join("tsjs_modules.rs"); @@ -160,30 +163,36 @@ fn main() { }); } -fn copy_bundle(filename: &str, required: bool, crate_dir: &Path, dist_dir: &Path, out_dir: &Path) { - let primary = dist_dir.join(filename); - let fallback = crate_dir.join("dist").join(filename); +fn bundle_sha256(path: &Path) -> String { + let content = fs::read(path).unwrap_or_else(|err| { + panic!( + "tsjs: failed to read copied bundle {} for hashing: {err}", + path.display() + ); + }); + hex::encode(Sha256::digest(&content)) +} + +fn copy_bundle(filename: &str, required: bool, dist_dir: &Path, out_dir: &Path) { + let source = dist_dir.join(filename); let target = out_dir.join(filename); - for source in [&primary, &fallback] { - if source.exists() { - if let Err(err) = fs::copy(source, &target) { - assert!( - !required, - "tsjs: failed to copy {} to {}: {err}", - source.display(), - target.display() - ); - } - return; + if source.exists() { + if let Err(err) = fs::copy(&source, &target) { + assert!( + !required, + "tsjs: failed to copy {} to {}: {err}", + source.display(), + target.display() + ); } + return; } assert!( !required, - "tsjs: bundle {filename} not found: {} (and fallback {}). Ensure Node is installed and `npm run build` succeeds, or commit dist/{filename}.", - primary.display(), - fallback.display() + "tsjs: bundle {filename} not found: {}. Ensure Node is installed and `npm run build` succeeds, or commit dist/{filename}.", + source.display() ); fs::write(&target, "").expect("should write optional empty bundle placeholder"); diff --git a/crates/trusted-server-js/src/bundle.rs b/crates/trusted-server-js/src/bundle.rs index 7ddc060ab..83815654a 100644 --- a/crates/trusted-server-js/src/bundle.rs +++ b/crates/trusted-server-js/src/bundle.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::sync::OnceLock; +use std::sync::{Mutex, MutexGuard, OnceLock}; use hex::encode; use sha2::{Digest as _, Sha256}; @@ -10,7 +10,7 @@ include!(concat!(env!("OUT_DIR"), "/tsjs_modules.rs")); #[must_use] #[inline] pub fn module_bundle(id: &str) -> Option<&'static str> { - module_map().get(id).copied() + module_meta_map().get(id).map(|module| module.bundle) } /// Return all available module IDs, in discovery order (core first). @@ -27,56 +27,159 @@ pub fn all_module_ids() -> Vec<&'static str> { #[must_use] #[inline] pub fn concatenate_modules(ids: &[&str]) -> String { - let map = module_map(); - let mut parts: Vec<&str> = Vec::new(); + let ordered_ids = concatenated_module_ids(ids); + let mut body = String::new(); + visit_concatenated_module_parts(&ordered_ids, |part| body.push_str(part)); + body +} + +/// SHA-256 hash of the concatenated modules, for cache-busting URLs. +/// +/// The hash is computed over the same byte sequence as [`concatenate_modules`] +/// without allocating that concatenated body. Results are cached by ordered +/// module ID list so HTML injection does not re-hash the full JS payload on +/// every page view. +#[must_use] +#[inline] +pub fn concatenated_hash(ids: &[&str]) -> String { + let key = concatenated_module_ids(ids); + if let Some(hash) = lock_concatenated_hash_cache().get(&key).cloned() { + return hash; + } + + let hash = hash_concatenated_modules(&key); + lock_concatenated_hash_cache().insert(key, hash.clone()); + hash +} + +/// SHA-256 hash of a single module's content (without prepending core). +/// +/// Used for cache-busting URLs of deferred modules served individually. +#[must_use] +#[inline] +pub fn single_module_hash(id: &str) -> Option<&'static str> { + module_meta_map().get(id).map(|module| module.sha256) +} + +fn concatenated_module_ids(ids: &[&str]) -> Vec<&'static str> { + let map = module_meta_map(); + let mut ordered = Vec::new(); - // Core always first if let Some(core) = map.get("core") { - parts.push(core); + ordered.push(core.id); } - // Then requested modules (excluding core, already included) for id in ids { if *id == "core" { continue; } - if let Some(bundle) = map.get(id) { - parts.push(bundle); + if let Some(module) = map.get(*id) { + ordered.push(module.id); } } - parts.join(";\n") + ordered } -/// SHA-256 hash of the concatenated modules, for cache-busting URLs. -#[must_use] -#[inline] -pub fn concatenated_hash(ids: &[&str]) -> String { - let body = concatenate_modules(ids); +fn hash_concatenated_modules(ids: &[&'static str]) -> String { let mut hasher = Sha256::new(); - hasher.update(body.as_bytes()); + visit_concatenated_module_parts(ids, |part| hasher.update(part.as_bytes())); encode(hasher.finalize()) } -/// SHA-256 hash of a single module's content (without prepending core). -/// -/// Used for cache-busting URLs of deferred modules served individually. -#[must_use] -#[inline] -pub fn single_module_hash(id: &str) -> Option { - module_bundle(id).map(|content| { - let mut hasher = Sha256::new(); - hasher.update(content.as_bytes()); - encode(hasher.finalize()) - }) +fn visit_concatenated_module_parts(ids: &[&'static str], mut visit: F) +where + F: FnMut(&'static str), +{ + let map = module_meta_map(); + let mut first = true; + + for id in ids { + let Some(module) = map.get(*id) else { + continue; + }; + if first { + first = false; + } else { + visit(";\n"); + } + visit(module.bundle); + } } -fn module_map() -> &'static HashMap<&'static str, &'static str> { - static MAP: OnceLock> = OnceLock::new(); +fn module_meta_map() -> &'static HashMap<&'static str, &'static TsjsModuleMeta> { + static MAP: OnceLock> = OnceLock::new(); MAP.get_or_init(|| { TSJS_MODULES .iter() - .map(|module| (module.id, module.bundle)) + .map(|module| (module.id, module)) .collect() }) } + +fn lock_concatenated_hash_cache() -> MutexGuard<'static, HashMap, String>> { + match concatenated_hash_cache().lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +fn concatenated_hash_cache() -> &'static Mutex, String>> { + static CACHE: OnceLock, String>>> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sha256_hex(bytes: &[u8]) -> String { + encode(Sha256::digest(bytes)) + } + + #[test] + fn generated_single_module_hashes_match_bundle_contents() { + for id in all_module_ids() { + let bundle = module_bundle(id).expect("should have module bundle"); + let generated_hash = single_module_hash(id).expect("should have generated hash"); + + assert_eq!( + generated_hash, + sha256_hex(bundle.as_bytes()), + "generated hash for module {id} should match included bundle bytes" + ); + } + } + + #[test] + fn concatenated_hash_matches_concatenated_bundle_contents() { + let available_ids = all_module_ids(); + let non_core_ids = available_ids + .iter() + .copied() + .filter(|id| *id != "core") + .take(3) + .collect::>(); + + let mut cases: Vec> = vec![Vec::new()]; + if let Some(first) = non_core_ids.first().copied() { + cases.push(vec![first]); + } + if non_core_ids.len() >= 2 { + cases.push(non_core_ids[..2].to_vec()); + cases.push(non_core_ids[..2].iter().rev().copied().collect()); + } + if non_core_ids.len() >= 3 { + cases.push(non_core_ids[..3].to_vec()); + } + + for ids in cases { + let concatenated = concatenate_modules(&ids); + assert_eq!( + concatenated_hash(&ids), + sha256_hex(concatenated.as_bytes()), + "concatenated hash should match concatenated bundle bytes for {ids:?}" + ); + } + } +} diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index ddb6544ce..9c59448fd 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -72,6 +72,7 @@ fail and the service will return its startup-error response. | `[ec]` | Edge Cookie (EC) ID generation | | `[tester_cookie]` | Optional tester-cookie endpoint | | `[proxy]` | Proxy SSRF allowlist and asset routes | +| `[cache]` | Static/rehosted asset cache policy rules | | `[image_optimizer]` | Reusable Image Optimizer profile sets | | `[request_signing]` | Ed25519 request signing | | `[auction]` | Auction orchestration | @@ -1031,6 +1032,78 @@ when_missing = "smart" See [Asset Routes](/guide/asset-routes) for request flow, S3 auth details, and Image Optimizer behavior. +## Cache Configuration + +Static and rehosted asset cache upgrades are operator-controlled. By default, +Trusted Server leaves arbitrary publisher-origin assets under origin cache +control. Add `[[cache.asset_rules]]` entries only for paths that are known to be +content-addressed or otherwise safe for the configured TTL. + +### `[[cache.asset_rules]]` + +Rules are evaluated in file order; the first enabled matching rule wins. +Disabled rules are ignored, which lets you keep framework presets documented in +config without enabling them for every publisher. + +| Field | Type | Required | Description | +| ---------------------------------- | ------------- | -------- | ----------------------------------------------------------------- | +| `id` | String | Yes | Unique operator-facing rule identifier | +| `enabled` | Boolean | No | Whether the rule participates in matching (default `false`) | +| `preset` | String | Matcher | Built-in preset such as `nextjs-static` | +| `path_prefix` | String | Matcher | Request path prefix | +| `path_glob` | String | Matcher | Single glob matched against the request path | +| `path_globs` | Array[String] | Matcher | Multiple globs matched against the request path | +| `path_regex` | String | Matcher | Regex matched against the request path | +| `extensions` | Array[String] | Matcher | Case-insensitive file extensions | +| `requires_hash_in_filename` | Boolean | No | Require an 8+ hex token in the final path segment before matching | +| `visibility` | String | No | `public` or `private` (default `public`) | +| `browser_ttl_seconds` | Integer | No | Browser `max-age` | +| `edge_ttl_seconds` | Integer | No | Edge/shared-cache TTL | +| `stale_while_revalidate_seconds` | Integer | No | Optional `stale-while-revalidate` | +| `stale_if_error_seconds` | Integer | No | Optional `stale-if-error` | +| `immutable` | Boolean | No | Add `immutable` when browser TTL is positive | + +Exactly one matcher must be configured per rule. `path_glob` and `path_globs` +are mutually exclusive. + +**Next.js preset example** (disabled until the publisher confirms +`/_next/static/` is content-addressed): + +```toml +[[cache.asset_rules]] +id = "nextjs-static" +enabled = false +preset = "nextjs-static" +visibility = "public" +browser_ttl_seconds = 31536000 +edge_ttl_seconds = 31536000 +immutable = true +``` + +**Publisher allowlist example**: + +```toml +[[cache.asset_rules]] +id = "publisher-fingerprinted-assets" +enabled = true +path_globs = [ + "/assets/**/*.js", + "/assets/**/*.css", + "/assets/**/*.png", + "/assets/**/*.webp", +] +requires_hash_in_filename = true +visibility = "public" +browser_ttl_seconds = 31536000 +edge_ttl_seconds = 31536000 +immutable = true +``` + +If `[cache]` is omitted or no enabled rule matches, Trusted Server preserves the +origin cache policy for publisher-origin assets. TS-owned validated hash URLs, +such as `/static/tsjs=...js?v=`, use their built-in cache policy and do +not require an asset rule. + ## Integration Configurations Settings for built-in integrations (Prebid, Next.js, Osano, Permutive, Testlight). For other diff --git a/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md b/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md new file mode 100644 index 000000000..f156aa768 --- /dev/null +++ b/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md @@ -0,0 +1,446 @@ +# Cache-Control Header Strategy Implementation Plan + +**Date:** 2026-07-06 +**Status:** Initial cache-header slice implemented in the current branch +**Spec:** `docs/superpowers/specs/2026-07-06-cache-control-header-design.md` + +## Scope + +Implement the **initial cache-header slice** from the current spec. The latest +spec resolves the initial-slice open questions and defers the larger dynamic +caching, template caching, streaming, and compression-offload work. + +Initial slice goals: + +1. Make TS-owned, hash-versioned TSJS responses cache correctly. +2. Make neutralized publisher Prebid compatibility responses safe to cache. +3. Add a structured, runtime-portable cache-policy model. +4. Add a configurable static/rehosted asset cache-rule engine so framework + assumptions are operator-controlled, not hard-coded. +5. Keep arbitrary publisher-origin assets origin-controlled unless an enabled + rule proves they are immutable-safe. + +Deferred follow-up features are listed separately below and should not be folded +into the initial cache-header PRs. + +## Decisions locked for the initial slice + +- SSAT-assembled HTML remains `Cache-Control: private, max-age=0` and strips + runtime edge-cache headers (`Surrogate-Control`, `Fastly-Surrogate-Control`, + `CDN-Cache-Control`, and `Cloudflare-CDN-Cache-Control`) whenever the ad stack + can inject per-user slot/bid state. +- TSJS keeps the current `/static/tsjs=...js?v=` canonical URL shape. + Matching hash/version requests receive immutable cache headers; missing or + mismatched hash/version requests keep short TTLs rather than redirecting. +- Runtime cache-key configuration must preserve the `v` query parameter for + `/static/tsjs=`. Fastly and Cloudflare include query strings in default cache + keys, but project-specific query normalization must not drop `v`. +- Framework-specific immutable paths, including Next.js `/_next/static/*`, must + be represented as configurable cache-rule presets. Do not add adapter- or + proxy-level hard-coded framework path checks. +- Operators decide which framework presets and publisher allowlists are enabled. + Arbitrary publisher CSS/JS/images remain origin-controlled unless an enabled + cache rule proves they are immutable-safe. +- TS-owned Prebid delivery is covered by deferred TSJS module URLs. Publisher + Prebid script URLs neutralized by TS are compatibility shims at stable URLs and + must use `no-store` or a very short TTL, not a year-long immutable policy. +- Rehosted assets are TS-owned copies once TS rewrites/hosts them. They should + use explicit normalized policies, with immutable only for TS-fingerprinted + rehosted URLs. +- Fastly and Cloudflare are the MVP runtime targets. Akamai mapping is deferred + until Akamai is on the roadmap. +- Dynamic HTML/RSC/API caching, dynamic `Vary`/cache-key normalization, + origin-template caching, transformed-template caching, true publisher-origin + streaming, parser-context bid splice, EdgeZero streaming parity, and SSAT HTML + compression offload are deferred follow-up features. +- All personalized/cookie-bearing response hardening in `response_privacy.rs` and + adapter middleware stays in place and runs after any new policy application. + +## Original baseline before this implementation + +| Area | Current file(s) | Baseline | +| ---------------------------- | --------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| TSJS URL injection | `crates/trusted-server-core/src/tsjs.rs` | Injects `/static/tsjs=...js?v=`; current branch is moving hash work out of the hot path. | +| TSJS serving | `publisher.rs`, `http_util.rs` | Historically served through `serve_static_with_etag` with 5-minute browser/edge TTLs. | +| Cache policy primitives | `cache_policy.rs` | Current branch adds typed policy rendering; still needs final alignment with no-store and config rules. | +| Neutralized Prebid shim | `crates/trusted-server-core/src/integrations/prebid.rs` | `handle_script_handler` currently returns an empty JS shim with `public, max-age=31536000`; this must be changed. | +| Cache privacy | `publisher.rs`, `response_privacy.rs`, adapter middleware | Ad-stack HTML and cookie-bearing responses are downgraded to private/shared-uncacheable. | +| Rehosted asset cache policy | `proxy.rs` | Policy is effectively origin-controlled or `no-store, private`; no normalized immutable/SWR policy. | +| Dynamic HTML/RSC/API caching | none | Deferred. No initial-slice `Vary` rewriting or Next router-header special casing. | +| Origin-template cache | none | Deferred. No cache API/override/template key/surrogate-key implementation exists. | + +## Definition of done for the initial slice + +- TSJS hash-version-matching requests emit one-year immutable browser cache and + one-year edge cache headers. +- TSJS missing/mismatched hash requests keep short TTL behavior. +- TSJS injected hash generation no longer concatenates and hashes the full + bundle on every page view. +- Neutralized publisher Prebid shim responses use `no-store` or a very short TTL. +- Cache policy is represented as structured data and can emit Fastly + `Surrogate-Control`, generic `CDN-Cache-Control`, Cloudflare-specific + `Cloudflare-CDN-Cache-Control`, and `s-maxage` fallback headers. +- Cache policy can represent `no-store`/uncacheable responses as well as public + and private TTL policies. +- TS config expresses static/rehosted cache policy through structured rules with + match criteria, policy fields, and `enabled` flags. +- Built-in framework presets, including Next.js `/_next/static/*`, are + implemented through the shared rule engine and can be disabled/overridden. +- Arbitrary publisher-origin assets remain origin-controlled unless matched by an + enabled preset or publisher allowlist. +- TS-owned rehosted assets have explicit normalized policies instead of blindly + passing through third-party defaults. +- MVP adapters emit the correct edge-cache header from shared policy: Fastly + `Surrogate-Control`, Cloudflare `CDN-Cache-Control` / + `Cloudflare-CDN-Cache-Control`, or portable `s-maxage` fallback. +- Deferred features are documented as deferred and are not accidentally + implemented as hard-coded Next.js/dynamic-cache behavior. +- Tests and target-matched checks pass for touched crates/adapters. + +## Proposed PR sequence + +### PR 1 — Structured cache policy primitives + +Status: implemented in the current branch. + +#### Code changes + +- Keep/add a core module such as `crates/trusted-server-core/src/cache_policy.rs`. +- Define structured policy types: + - `CacheVisibility::{Public, Private}` + - `CachePolicy { visibility, browser_ttl, edge_ttl, stale_while_revalidate, +stale_if_error, immutable }` + - a `no-store` / uncacheable representation, either as a policy mode or a + dedicated helper, so neutralized shims and error responses do not need + ad-hoc strings; + - `EdgeCacheHeader::{SurrogateControl, CdnCacheControl, +CloudflareCdnCacheControl, SMaxageFallback, None}`. +- Add helpers that render policy into headers: + - browser `Cache-Control` + - Fastly `Surrogate-Control` + - generic `CDN-Cache-Control` + - Cloudflare-specific `Cloudflare-CDN-Cache-Control` + - portable `s-maxage` fallback. +- Keep helpers side-effect-limited: they should only mutate cache headers they + own and should not bypass `response_privacy` hardening. When applying private + or no-store policies, remove any existing edge-cache headers owned by the + helper so stale `Surrogate-Control`/CDN cache headers cannot survive. +- Add default policy constructors/constants for: + - immutable static; + - short TSJS fallback; + - neutralized Prebid shim (`no-store` or very short TTL); + - uncacheable private. + +#### Tests + +- Unit-test exact header rendering for immutable, short edge/browser split, + private, no-store, SWR/SIE, generic CDN, Cloudflare-specific CDN, and fallback + `s-maxage` policies. +- Test that `immutable` is omitted when browser TTL is absent or zero. +- Test that edge-header output is disabled for private/no-store responses, and + that applying private/no-store removes any pre-existing edge-cache header the + helper owns. + +### PR 2 — TSJS immutable hash-version serving + +Status: implemented in the current branch with runtime-specific edge-header +selection. + +#### Code changes + +- Extend `crates/trusted-server-js/build.rs` generated metadata with per-module + SHA-256 hashes. +- Update `trusted-server-js/src/bundle.rs`: + - `single_module_hash(id)` returns generated hash instead of hashing content; + - `concatenated_hash(ids)` hashes incrementally without concatenating a full + `String`, or caches the result per normalized module-id set; + - `concatenate_modules(ids)` can remain for serving the response body. +- Update `handle_tsjs_dynamic` in `publisher.rs`: + - parse `?v=` from the request URI; + - compare it with the canonical hash for the requested bundle; + - if it matches, apply immutable static policy plus `Vary: Accept-Encoding`, + ETag, and `X-Compress-Hint: on`; + - if missing/mismatched, keep short TTL policy plus ETag and + `X-Compress-Hint: on`. +- Keep the current canonical path shape (`/static/tsjs=...js?v=`). +- Document/verify that runtime cache-key configuration preserves the `v` query + parameter for `/static/tsjs=`. + +#### Tests + +- `tsjs_script_src` and deferred script tests still produce `?v=`. +- Matching `?v=` returns: + - `Cache-Control: public, max-age=31536000, immutable` + - runtime edge header via policy helper; + - `Vary: Accept-Encoding`; + - ETag. +- Missing/mismatched `?v=` returns short TTL and no `immutable`. +- Deferred disabled module still 404s. +- Hash helpers do not allocate the concatenated body just to hash it. + +### PR 3 — Neutralized publisher Prebid shim cache safety + +Fix the stable publisher Prebid compatibility route separately from TS-owned +Prebid delivery. + +#### Code changes + +- Update `PrebidIntegration::handle_script_handler` in + `crates/trusted-server-core/src/integrations/prebid.rs`. +- Replace the current year-long `public, max-age=31536000` response with either: + - `Cache-Control: no-store`, preferred for compatibility when integration + enablement/config can change; or + - a very short TTL if no-store is too conservative. +- Ensure no `Surrogate-Control`/CDN edge header is emitted for the neutralized + stable URL. +- Keep TS-owned Prebid bundle delivery on the deferred TSJS module path, where + matching `?v=` remains immutable. + +#### Tests + +- Neutralized Prebid script handler returns the empty compatibility script with + `no-store` or the chosen short TTL. +- Neutralized Prebid shim does not emit immutable or year-long cache headers. +- TSJS deferred Prebid still receives immutable headers when `?v=` matches. + +### PR 4 — Configurable static asset cache-rule engine + +Introduce operator-configurable static asset rules before applying immutable +upgrades to publisher-origin assets. + +#### Code changes + +- Add cache-rule settings rather than hard-coded path checks. Suggested shape: + - `CacheAssetRule { id, enabled, matcher, policy }` + - `CacheAssetMatcher::{PathPrefix, Glob, Regex, Extension, Preset}` + - `CacheAssetPreset::NextJsStatic` expands to `/_next/static/*` when enabled. +- Add cache settings under `Settings` (and `trusted-server.example.toml`) with + `#[serde(deny_unknown_fields)]` validation consistent with the rest of the + config model. +- Add a shared rule evaluator with deterministic precedence. Prefer an ordered + rule list where the first enabled match wins; reject duplicate rule IDs and + invalid matcher combinations during settings validation. +- Ship framework presets as data/config defaults or documented examples, not as + special cases in proxy/adapters. +- The Next.js preset may be present in example config, but operators must be able + to disable/override it. Do not silently apply it through a hard-coded branch. +- Support publisher-defined allowlist rules for other frameworks or + publisher-specific fingerprinted paths. +- Apply immutable policy only when an enabled rule/preset says the URL is + content-addressed, or for TS-owned validated hash URLs such as TSJS. + +#### Tests + +- With the Next.js preset enabled, `/_next/static/*` gets immutable policy. +- With the Next.js preset disabled, the same `/_next/static/*` remains + origin-controlled. +- Publisher-defined allowlist rule can mark a non-Next fingerprinted path + immutable. +- Non-matching publisher asset remains origin-controlled. +- Rule precedence is deterministic. +- Invalid regex/glob/config fails validation clearly. + +### PR 5 — MVP runtime edge-header mapping and docs + +Make the shared policy output explicit per runtime before wiring the rule engine +into more routes. This prevents new code from copying the current core +Fastly-specific `Surrogate-Control` behavior. + +#### Code changes + +- Stop requiring core helpers such as `handle_tsjs_dynamic` or + `serve_static_with_etag` to hard-code Fastly's `Surrogate-Control`. +- Choose one adapter boundary pattern and use it consistently: + - pass the runtime `EdgeCacheHeader`/policy emitter into core handlers; or + - return cache-policy metadata in response extensions and let adapters render + runtime-specific headers after route handling. +- Fastly adapter emits `Surrogate-Control` for edge TTLs. +- Cloudflare adapter emits `CDN-Cache-Control` or + `Cloudflare-CDN-Cache-Control`, depending on the chosen adapter convention. +- Portable/local fallback can use `s-maxage` inside `Cache-Control` when no + runtime-specific edge header is available. +- Akamai mapping remains absent/deferred; do not add untested Akamai behavior. +- Update `trusted-server.example.toml` and docs with disabled framework preset + examples and operator-owned allowlist examples. + +#### Tests + +- Fastly TSJS/static policy application emits `Surrogate-Control`. +- Cloudflare TSJS/static policy application emits the selected Cloudflare CDN + cache header and does not emit Fastly-only `Surrogate-Control`. +- Fallback policy emits `s-maxage` only for public/shared-cacheable responses. +- Private/no-store responses remove or avoid all edge-cache headers. + +### PR 6 — Apply static/rehosted policies to proxy responses + +Wire the rule engine into the routes that emit publisher-origin or rehosted +assets, using the runtime edge-header mapping from PR 5. + +#### Code changes + +- Extend `AssetProxyCachePolicy` in `proxy.rs` beyond + `OriginControlled`/`NoStorePrivate`, for example: + - `OriginControlled` + - `NoStorePrivate` + - `Normalized(CachePolicy)` from a matched enabled rule. +- Apply normalized policy after route finalization but before final response + privacy hardening. +- Preserve existing no-store/private handling for errors, signed failures, or + responses that set cookies/security headers. +- Ensure operator `response_headers` cannot weaken protected private/no-store + decisions. +- For TS-owned rehosted copies: + - use immutable only for fingerprinted TS-owned URLs; + - use conservative edge/browser TTLs for stable rehosted URLs; + - keep dynamic/personalized endpoints uncached. + +#### Tests + +- Rehosted/fingerprinted route matched by an enabled rule gets immutable policy. +- Stable rehosted route gets the configured conservative policy, not a borrowed + third-party `no-store` unless configured. +- Rehosted error responses keep `no-store, private`. +- `Set-Cookie` response remains private/no-store and loses surrogate headers. +- Operator response headers cannot re-enable shared caching for protected + responses. + +## Initial config sketch + +Exact names can change during implementation, but keep the shape structured and +operator-controlled. + +```toml +[[cache.asset_rules]] +id = "nextjs-static" +enabled = false # operators may enable for Next.js publishers +preset = "nextjs-static" +visibility = "public" +browser_ttl_seconds = 31536000 +edge_ttl_seconds = 31536000 +immutable = true + +[[cache.asset_rules]] +id = "publisher-fingerprinted-assets-example" +enabled = false +path_globs = [ + "/assets/**/*.js", + "/assets/**/*.css", + "/assets/**/*.png", + "/assets/**/*.jpg", + "/assets/**/*.webp", + "/assets/**/*.avif", +] +requires_hash_in_filename = true +visibility = "public" +browser_ttl_seconds = 31536000 +edge_ttl_seconds = 31536000 +immutable = true + +[cache.tsjs.versioned] +visibility = "public" +browser_ttl_seconds = 31536000 +edge_ttl_seconds = 31536000 +immutable = true + +[cache.tsjs.fallback] +visibility = "public" +browser_ttl_seconds = 300 +edge_ttl_seconds = 300 +stale_while_revalidate_seconds = 60 +stale_if_error_seconds = 86400 + +[cache.prebid_neutralized] +mode = "no-store" +``` + +Defaults should preserve current behavior unless a rule is explicitly enabled or +unless the response is TS-owned and hash-validated, such as TSJS. + +## Deferred follow-up backlog + +These remain valuable, but are intentionally outside the initial cache-header +slice. + +| Follow-up | Why deferred | Notes | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| True publisher-origin streaming | Requires platform/body boundary changes and adapter streaming semantics | Includes avoiding full `take_body_bytes()` materialization on Fastly and documenting/implementing non-Fastly streaming parity. | +| Parser-context bid splice | Requires HTML pipeline redesign | Replace raw `` | `Cache-Control: public, max-age=31536000, immutable` plus runtime edge header | The serving path must validate that `v` matches the bytes served. | +| TSJS missing/mismatched `?v=` | Short TTL or redirect to canonical hashed URL | Do not mark immutable. | +| TSJS deferred modules, including Prebid | Same as TSJS hash-matching policy | Example: `/static/tsjs=tsjs-prebid.min.js?v=`. | +| Publisher Prebid URL neutralized by TS | `no-store` or very short TTL | The empty compatibility shim is config-dependent and served at a stable publisher URL. Do not cache it for a year. | +| Enabled framework preset static, e.g. Next.js `/_next/static/*` | `Cache-Control: public, max-age=31536000, immutable` | Applied through configurable preset/allowlist rules. | +| TS-fingerprinted rehosted asset | `Cache-Control: public, max-age=31536000, immutable` | Safe only when TS owns the fingerprinted URL. | +| Stable TS-owned/rehosted URL | Conservative short browser TTL, optional longer edge TTL | Do not use `immutable`. | +| Arbitrary publisher-origin CSS/JS/images | Origin-controlled by default | TS may upgrade only via enabled framework preset or publisher allowlist. | +| SSAT-assembled ad-stack HTML | `Cache-Control: private, max-age=0`; strip runtime edge-cache headers | Must never enter shared cache because it can contain per-user slot/bid data. | +| Dynamic HTML/RSC/API | Origin-controlled in this slice | Future dynamic caching belongs to #859. | + +## TSJS-specific requirements + +Current TSJS URLs already include a content hash query string, for example: + +```text +/static/tsjs=tsjs-unified.min.js?v= +/static/tsjs=tsjs-prebid.min.js?v= +``` + +The current serving path still emits a short cache policy. Update it so that: + +- hash-matching requests emit one-year immutable browser caching; +- hash-matching requests emit the runtime edge header with equivalent long edge TTL; +- missing or mismatched hash requests do not receive immutable caching; +- cache-key configuration preserves the `v` query parameter; +- TSJS hashes used in injected URLs are generated at build time or cached so HTML injection does not re-concatenate and re-hash large bundles per pageview; +- `Vary: Accept-Encoding` remains on compressed/static responses; +- ETags may remain as a fallback for clients or intermediaries that revalidate anyway. + +Fastly and Cloudflare include query strings in default cache keys, but TS must still avoid any project-specific query normalization that drops `v` for `/static/tsjs=`. + +## SSAT HTML privacy requirement + +SSAT-assembled ad-stack HTML can contain per-user data such as slot state or bid data. It must remain: + +```http +Cache-Control: private, max-age=0 +``` + +and must strip runtime edge-cache headers, including: + +```http +Surrogate-Control +Fastly-Surrogate-Control +CDN-Cache-Control +Cloudflare-CDN-Cache-Control +``` + +This requirement applies to the browser-facing assembled response. Origin-template caching is separate follow-up work in #859. + +## Runtime header mapping for MVP + +Adapters should render the shared policy as follows: + +| Runtime | Edge/shared-cache header | +| ----------------- | ---------------------------------------------------- | +| Fastly | `Surrogate-Control` | +| Cloudflare | `CDN-Cache-Control` / `Cloudflare-CDN-Cache-Control` | +| Portable fallback | `s-maxage` in `Cache-Control` | + +Akamai mapping is deferred until Akamai is on the roadmap. + +## Acceptance criteria + +- [ ] Cache policy is represented as structured fields, not hard-coded header strings. +- [ ] Built-in framework presets, including a disableable/overrideable Next.js `/_next/static/*` rule, are implemented through the shared cache-policy rule engine. +- [ ] TSJS hash-matching requests for unified and deferred modules emit `public, max-age=31536000, immutable` plus the runtime edge header. +- [ ] TSJS missing/mismatched hash requests do not get immutable caching. +- [ ] TSJS hash generation is build-time or cached enough that HTML injection does not re-concatenate/re-hash the bundle per pageview. +- [ ] Runtime cache-key configuration preserves the `v` query parameter for `/static/tsjs=`. +- [ ] Neutralized publisher Prebid shim responses use `no-store` or a short TTL, not a year-long policy. +- [ ] Arbitrary publisher-origin assets remain origin-controlled unless covered by an enabled framework preset or publisher allowlist. +- [ ] TS-owned rehosted assets have explicit normalized cache policy; immutable is used only for TS-fingerprinted rehosted URLs. +- [ ] SSAT-assembled ad-stack HTML continues to emit `private, max-age=0` and strips all runtime edge-cache headers. +- [ ] Fastly and Cloudflare adapters emit the correct edge-cache header from the shared policy, with portable `s-maxage` fallback where needed. +- [ ] Dynamic HTML/RSC/API Vary/cache-key normalization is not hard-coded in this PR and is deferred to #859. +- [ ] SSAT streaming fixes and compression offload are not included in this PR and remain tracked by #857 and #858. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 19ecda4a5..2b46336ab 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -116,6 +116,27 @@ enabled = false # Required for integrations.prebid.external_bundle_url and first-party proxy redirects. # allowed_domains = ["ads.example.com", "assets.example.com", "*.cdn.example.com"] +# Static/rehosted asset cache policies are operator-controlled. Keep framework +# presets disabled unless the matched publisher paths are known content-addressed. +# [[cache.asset_rules]] +# id = "nextjs-static" +# enabled = false +# preset = "nextjs-static" +# visibility = "public" +# browser_ttl_seconds = 31536000 +# edge_ttl_seconds = 31536000 +# immutable = true +# +# [[cache.asset_rules]] +# id = "publisher-fingerprinted-assets" +# enabled = false +# path_globs = ["/assets/**/*.js", "/assets/**/*.css", "/assets/**/*.png", "/assets/**/*.webp"] +# requires_hash_in_filename = true +# visibility = "public" +# browser_ttl_seconds = 31536000 +# edge_ttl_seconds = 31536000 +# immutable = true + [auction] enabled = false # Defaults to true. Keep this leaf present when using the EdgeZero v0.0.4 From d6328f42929d7aeade9574afefa288eca24ba26b Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 8 Jul 2026 12:13:01 -0500 Subject: [PATCH 280/395] Format cache configuration docs --- docs/guide/configuration.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 9c59448fd..258a9a4bc 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1045,23 +1045,23 @@ Rules are evaluated in file order; the first enabled matching rule wins. Disabled rules are ignored, which lets you keep framework presets documented in config without enabling them for every publisher. -| Field | Type | Required | Description | -| ---------------------------------- | ------------- | -------- | ----------------------------------------------------------------- | -| `id` | String | Yes | Unique operator-facing rule identifier | -| `enabled` | Boolean | No | Whether the rule participates in matching (default `false`) | -| `preset` | String | Matcher | Built-in preset such as `nextjs-static` | -| `path_prefix` | String | Matcher | Request path prefix | -| `path_glob` | String | Matcher | Single glob matched against the request path | -| `path_globs` | Array[String] | Matcher | Multiple globs matched against the request path | -| `path_regex` | String | Matcher | Regex matched against the request path | -| `extensions` | Array[String] | Matcher | Case-insensitive file extensions | -| `requires_hash_in_filename` | Boolean | No | Require an 8+ hex token in the final path segment before matching | -| `visibility` | String | No | `public` or `private` (default `public`) | -| `browser_ttl_seconds` | Integer | No | Browser `max-age` | -| `edge_ttl_seconds` | Integer | No | Edge/shared-cache TTL | -| `stale_while_revalidate_seconds` | Integer | No | Optional `stale-while-revalidate` | -| `stale_if_error_seconds` | Integer | No | Optional `stale-if-error` | -| `immutable` | Boolean | No | Add `immutable` when browser TTL is positive | +| Field | Type | Required | Description | +| -------------------------------- | ------------- | -------- | ----------------------------------------------------------------- | +| `id` | String | Yes | Unique operator-facing rule identifier | +| `enabled` | Boolean | No | Whether the rule participates in matching (default `false`) | +| `preset` | String | Matcher | Built-in preset such as `nextjs-static` | +| `path_prefix` | String | Matcher | Request path prefix | +| `path_glob` | String | Matcher | Single glob matched against the request path | +| `path_globs` | Array[String] | Matcher | Multiple globs matched against the request path | +| `path_regex` | String | Matcher | Regex matched against the request path | +| `extensions` | Array[String] | Matcher | Case-insensitive file extensions | +| `requires_hash_in_filename` | Boolean | No | Require an 8+ hex token in the final path segment before matching | +| `visibility` | String | No | `public` or `private` (default `public`) | +| `browser_ttl_seconds` | Integer | No | Browser `max-age` | +| `edge_ttl_seconds` | Integer | No | Edge/shared-cache TTL | +| `stale_while_revalidate_seconds` | Integer | No | Optional `stale-while-revalidate` | +| `stale_if_error_seconds` | Integer | No | Optional `stale-if-error` | +| `immutable` | Boolean | No | Add `immutable` when browser TTL is positive | Exactly one matcher must be configured per rule. `path_glob` and `path_globs` are mutually exclusive. From 0e413c2b62e33e9dd29c7dcdacd0b97ff6362ce5 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 15 Jul 2026 11:44:46 -0500 Subject: [PATCH 281/395] Address cache policy review feedback --- .../wrangler.ci.toml | 3 + .../wrangler.toml | 3 + .../trusted-server-adapter-fastly/src/app.rs | 2 +- .../trusted-server-adapter-fastly/src/main.rs | 7 +- .../src/middleware.rs | 27 +- .../src/integrations/gpt_diagnostics.rs | 4 +- crates/trusted-server-core/src/proxy.rs | 51 +++- crates/trusted-server-core/src/publisher.rs | 2 +- .../src/response_privacy.rs | 58 +++- crates/trusted-server-core/src/settings.rs | 282 ++++++++++++++++-- docs/guide/configuration.md | 95 ++++-- ...ache-control-header-implementation-plan.md | 34 ++- .../2026-07-06-cache-control-header-design.md | 54 ++-- trusted-server.example.toml | 8 +- 14 files changed, 514 insertions(+), 116 deletions(-) diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml index e6891eb79..88a2dfc74 100644 --- a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml +++ b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml @@ -6,6 +6,9 @@ compatibility_date = "2024-09-23" compatibility_flags = ["nodejs_compat", "cache_option_enabled"] # No [build] section — bundle is pre-built in CI; wrangler dev must not rebuild. +[cache] +enabled = true + [[kv_namespaces]] binding = "TRUSTED_SERVER_KV" id = "ci-local-kv" diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.toml b/crates/trusted-server-adapter-cloudflare/wrangler.toml index 7c91173fc..9acdb13ab 100644 --- a/crates/trusted-server-adapter-cloudflare/wrangler.toml +++ b/crates/trusted-server-adapter-cloudflare/wrangler.toml @@ -11,6 +11,9 @@ compatibility_date = "2024-09-23" # (auction-eligible publisher navigations), so this is a hard requirement. compatibility_flags = ["nodejs_compat", "cache_option_enabled"] +[cache] +enabled = true + [build] command = "bash build.sh" diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index ca93eb3c2..7991f488d 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -96,7 +96,7 @@ use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; -use trusted_server_core::auction::{build_orchestrator, AuctionOrchestrator}; +use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use trusted_server_core::ec::EcContext; diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 24be7ad20..e28c0726f 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -333,10 +333,11 @@ fn send_edgezero_response( effects.apply_to_response(&mut response); } - // Final cache guard: EC finalization and request-filter effects may have - // added a per-user Set-Cookie after `apply_finalize_headers` ran, so - // re-apply the privacy downgrade before send. + // Final cache guards: EC finalization and request-filter effects may have + // added a per-user Set-Cookie or a private/no-store directive after + // `apply_finalize_headers` and normalized asset policy reapplication ran. crate::middleware::enforce_set_cookie_cache_privacy(&mut response); + crate::middleware::enforce_uncacheable_cache_privacy(&mut response); let (parts, body) = response.into_parts(); diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 2c00ac2ff..153d90295 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -235,7 +235,9 @@ pub(crate) fn apply_finalize_headers( /// entry point (`main.rs`) can re-apply it after /// [`ec_finalize_response`](trusted_server_core::ec::finalize::ec_finalize_response) /// writes the EC identity `Set-Cookie`, using the single shared implementation. -pub(crate) use trusted_server_core::response_privacy::enforce_set_cookie_cache_privacy; +pub(crate) use trusted_server_core::response_privacy::{ + enforce_set_cookie_cache_privacy, enforce_uncacheable_cache_privacy, +}; // --------------------------------------------------------------------------- // Tests @@ -496,6 +498,29 @@ mod tests { ); } + #[test] + fn enforce_uncacheable_cache_privacy_handles_late_filter_headers() { + let mut response = response_with_headers(&[ + ("cache-control", "private, max-age=0"), + ("surrogate-control", "max-age=600"), + ]); + + enforce_uncacheable_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "should preserve the late private directive" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip the normalized edge header after late filter effects" + ); + } + // --------------------------------------------------------------------------- // FinalizeResponseMiddleware::handle tests // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 5bd86d19e..3e63f3d35 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -12,9 +12,9 @@ use validator::Validate; use edgezero_core::body::Body as EdgeBody; +use crate::cache_policy::EDGE_CACHE_HEADER_NAMES; use crate::error::TrustedServerError; use crate::http_util::is_navigation_request; -use crate::response_privacy::CDN_CACHE_HEADERS; use crate::settings::{IntegrationConfig, Settings}; use crate::tsjs; @@ -257,7 +257,7 @@ pub fn finalize_response( header::CACHE_CONTROL, HeaderValue::from_static("private, no-store"), ); - for name in CDN_CACHE_HEADERS { + for name in EDGE_CACHE_HEADER_NAMES { response.headers_mut().remove(*name); } } diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 0270c4292..b51cd75f6 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -14,7 +14,9 @@ use std::time::Duration; use web_time::{SystemTime, UNIX_EPOCH}; use crate::cache_policy::{ - apply_no_store_private_to_headers, CachePolicy, EdgeCacheHeader, NO_STORE_PRIVATE_CACHE_CONTROL, + CachePolicy, EdgeCacheHeader, NO_STORE_PRIVATE_CACHE_CONTROL, + apply_no_store_private_to_headers, cache_control_headers_are_private_or_no_store, + remove_edge_cache_headers, }; use crate::constants::{ HEADER_ACCEPT, HEADER_ACCEPT_ENCODING, HEADER_ACCEPT_LANGUAGE, HEADER_REFERER, @@ -127,7 +129,11 @@ impl AssetProxyCachePolicy { Self::OriginControlled => {} Self::NoStorePrivate => apply_no_store_cache_control(response), Self::Normalized(policy) => { - policy.apply_to_headers(response.headers_mut(), edge_header) + if cache_control_headers_are_private_or_no_store(response.headers()) { + remove_edge_cache_headers(response.headers_mut()); + } else { + policy.apply_to_headers(response.headers_mut(), edge_header); + } } } } @@ -4329,7 +4335,7 @@ mod tests { } #[test] - fn handle_asset_proxy_request_applies_configured_normalized_cache_policy() { + fn handle_asset_proxy_request_replaces_third_party_cache_policy_for_rehosted_asset() { futures::executor::block_on(async { let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( @@ -4379,7 +4385,7 @@ mod tests { assert_eq!( response_header(&response, header::CACHE_CONTROL), Some("public, max-age=31536000, immutable"), - "core response should apply browser cache policy immediately" + "configured rehost policy should replace the third-party no-store directive" ); assert!( response.headers().get("surrogate-control").is_none(), @@ -4401,6 +4407,43 @@ mod tests { }); } + #[test] + fn normalized_asset_policy_preserves_final_private_or_no_store_directives() { + for cache_control in ["private, max-age=0", "no-store"] { + let mut response = edge_response_builder() + .header(header::CACHE_CONTROL, cache_control) + .header("surrogate-control", "max-age=31536000") + .header("cdn-cache-control", "max-age=31536000") + .header("cloudflare-cdn-cache-control", "max-age=31536000") + .body(EdgeBody::empty()) + .expect("should build asset response"); + + AssetProxyCachePolicy::Normalized(CachePolicy::public_immutable(Duration::from_secs( + 31_536_000, + ))) + .apply_after_route_finalization(&mut response, EdgeCacheHeader::SurrogateControl); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "final privacy directive should veto normalized cache policy" + ); + assert!( + [ + "surrogate-control", + "cdn-cache-control", + "cloudflare-cdn-cache-control", + ] + .iter() + .all(|name| !response.headers().contains_key(*name)), + "final privacy directive should remove every edge-cache header" + ); + } + } + #[test] fn handle_asset_proxy_request_leaves_non_matching_assets_origin_controlled() { futures::executor::block_on(async { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1da702196..674d3d718 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -48,7 +48,7 @@ use crate::auction::types::{ AuctionContext, AuctionRequest, Bid, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, }; use crate::cache_policy::{ - cache_control_headers_are_private_or_no_store, CachePolicy, EdgeCacheHeader, + CachePolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, }; use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 40650d7e7..8ccda97bd 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -17,11 +17,6 @@ use crate::cache_policy::{ }; use crate::settings::Settings; -/// Runtime edge-cache headers stripped from private or cookie-bearing responses. -pub use crate::cache_policy::EDGE_CACHE_HEADER_NAMES as SURROGATE_CACHE_HEADERS; -/// Backwards-compatible name used by integrations that clear every edge-cache directive. -pub use crate::cache_policy::EDGE_CACHE_HEADER_NAMES as CDN_CACHE_HEADERS; - fn cache_control_is_private_or_no_store(response: &Response) -> bool { cache_control_headers_are_private_or_no_store(response.headers()) } @@ -38,6 +33,17 @@ pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { response.headers_mut().remove(header::LAST_MODIFIED); } +/// Removes runtime edge-cache headers from a response finalized as uncacheable. +/// +/// Call this after any late response-header mutations so a final `private` or +/// `no-store` directive cannot coexist with an independently authoritative edge +/// cache header. +pub fn enforce_uncacheable_cache_privacy(response: &mut Response) { + if cache_control_is_private_or_no_store(response) { + remove_edge_cache_headers(response.headers_mut()); + } +} + /// Forces cookie-bearing responses to stay private to shared caches. /// /// Any response that sets a per-user cookie (notably the EC identity cookie) @@ -85,9 +91,7 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: enforce_set_cookie_cache_privacy(response); let response_is_uncacheable = cache_control_is_private_or_no_store(response); - if response_is_uncacheable { - remove_edge_cache_headers(response.headers_mut()); - } + enforce_uncacheable_cache_privacy(response); for (key, value) in &settings.response_headers { if response_is_uncacheable @@ -113,9 +117,7 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: response.headers_mut().insert(header_name, header_value); } - if cache_control_is_private_or_no_store(response) { - remove_edge_cache_headers(response.headers_mut()); - } + enforce_uncacheable_cache_privacy(response); // Operator headers can themselves introduce Set-Cookie (alongside public // edge-cache headers) onto a previously cookieless response, which the @@ -130,6 +132,8 @@ mod tests { use edgezero_core::http::response_builder; + use crate::cache_policy::EDGE_CACHE_HEADER_NAMES; + fn settings_with_response_headers(headers: &[(&str, &str)]) -> Settings { let mut s = Settings::from_toml( r#" @@ -178,7 +182,7 @@ mod tests { ); for header_name in [header::ETAG.as_str(), header::LAST_MODIFIED.as_str()] .into_iter() - .chain(SURROGATE_CACHE_HEADERS.iter().copied()) + .chain(EDGE_CACHE_HEADER_NAMES.iter().copied()) { assert!( !response.headers().contains_key(header_name), @@ -307,7 +311,7 @@ mod tests { "private, no-store", "operator cache headers must not weaken an existing private response" ); - for header_name in SURROGATE_CACHE_HEADERS { + for header_name in EDGE_CACHE_HEADER_NAMES { assert!( !response.headers().contains_key(*header_name), "operator headers must not restore shared caching through {header_name}" @@ -341,6 +345,34 @@ mod tests { ); } + #[test] + fn final_uncacheable_guard_strips_edge_headers_without_a_cookie() { + let mut response = response_builder() + .header(header::CACHE_CONTROL, "no-store") + .header("surrogate-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + enforce_uncacheable_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store"), + "final guard should preserve the uncacheable directive" + ); + assert!( + EDGE_CACHE_HEADER_NAMES + .iter() + .all(|name| !response.headers().contains_key(*name)), + "final guard should remove every edge-cache header" + ); + } + #[test] fn applies_operator_headers_on_cookieless_response() { let settings = settings_with_response_headers(&[("x-operator", "value")]); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index ad17595f9..effbf613f 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1870,7 +1870,7 @@ fn validate_tinybird_secret(value: &str, setting: &str) -> Result<(), Report Result<(), Report> { let mut seen_ids = HashSet::new(); for rule in &self.asset_rules { @@ -1904,6 +1904,8 @@ impl CacheSettings { message: format!("cache.asset_rules contains duplicate id `{}`", rule.id), })); } + } + for rule in &self.asset_rules { rule.prepare_runtime()?; } Ok(()) @@ -1955,7 +1957,7 @@ pub struct CacheAssetRule { /// File extensions matched against the request path, case-insensitively. #[serde(default)] pub extensions: Vec, - /// Require a hash-like token in the final path segment before the rule matches. + /// Require a supported bundler fingerprint suffix in the filename before matching. #[serde(default)] pub requires_hash_in_filename: bool, /// Browser-facing cache visibility. @@ -2015,9 +2017,14 @@ impl CacheAssetRule { } fn prepare_runtime(&self) -> Result<(), Report> { + if !self.enabled { + return Ok(()); + } + self.validate_matcher_shape()?; self.compiled_regex().map(|_| ())?; self.compiled_globs().map(|_| ())?; + self.validate_policy_shape()?; Ok(()) } @@ -2048,6 +2055,46 @@ impl CacheAssetRule { Ok(()) } + fn validate_policy_shape(&self) -> Result<(), Report> { + if self.browser_ttl_seconds.is_none() && self.edge_ttl_seconds.is_none() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` must configure browser_ttl_seconds or edge_ttl_seconds", + self.id + ), + })); + } + + if !self.immutable { + return Ok(()); + } + + if self + .browser_ttl_seconds + .is_none_or(|browser_ttl| browser_ttl == 0) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` sets immutable without a positive browser_ttl_seconds", + self.id + ), + })); + } + + let preset_is_content_addressed = + matches!(self.preset, Some(CacheAssetPreset::NextJsStatic)); + if !preset_is_content_addressed && !self.requires_hash_in_filename { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` sets immutable without requires_hash_in_filename or a content-addressed preset", + self.id + ), + })); + } + + Ok(()) + } + fn compiled_regex(&self) -> Result, Report> { let Some(pattern) = self.path_regex.as_deref() else { return Ok(None); @@ -2094,13 +2141,22 @@ impl CacheAssetRule { } fn matches_path(&self, path: &str) -> Result> { - if !self.enabled { + if !self.enabled || !self.matcher_matches_path(path)? { return Ok(false); } - if self.requires_hash_in_filename && !filename_contains_hash(path) { + + if self.requires_hash_in_filename && !filename_contains_fingerprint(path) { + log::debug!( + "cache asset rule `{}` rejects path `{path}` because the filename has no supported fingerprint", + self.id + ); return Ok(false); } + Ok(true) + } + + fn matcher_matches_path(&self, path: &str) -> Result> { if let Some(preset) = self.preset { return Ok(preset.matches_path(path)); } @@ -2178,11 +2234,44 @@ fn path_extension(path: &str) -> Option { (!extension.is_empty()).then(|| extension.to_ascii_lowercase()) } -fn filename_contains_hash(path: &str) -> bool { +fn filename_contains_fingerprint(path: &str) -> bool { let filename = path.rsplit('/').next().unwrap_or(path); - filename - .split(['.', '-', '_', '~']) - .any(|segment| segment.len() >= 8 && segment.chars().all(|ch| ch.is_ascii_hexdigit())) + let Some((stem, extension)) = filename.rsplit_once('.') else { + return false; + }; + if stem.is_empty() || extension.is_empty() { + return false; + } + + stem.char_indices() + .filter(|(_, ch)| matches!(ch, '.' | '-' | '_' | '~')) + .any(|(separator_index, separator)| { + let candidate_start = separator_index + separator.len_utf8(); + let prefix = &stem[..separator_index]; + let candidate = &stem[candidate_start..]; + !prefix.is_empty() && fingerprint_candidate_is_supported(candidate) + }) +} + +fn fingerprint_candidate_is_supported(candidate: &str) -> bool { + let is_hex = candidate.len() >= 8 + && candidate.chars().all(|ch| ch.is_ascii_hexdigit()) + && candidate.chars().any(|ch| ch.is_ascii_alphabetic()); + let is_esbuild_base32 = candidate.len() == 8 + && candidate + .chars() + .all(|ch| ch.is_ascii_uppercase() || matches!(ch, '2'..='7')) + && candidate.chars().any(|ch| ch.is_ascii_alphabetic()); + let is_vite_base64url = candidate.len() == 8 + && candidate + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_')) + && candidate.chars().any(|ch| ch.is_ascii_uppercase()) + && candidate + .chars() + .any(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || matches!(ch, '-' | '_')); + + is_hex || is_esbuild_base32 || is_vite_base64url } /// Debug-only features. All flags default to `false` (off in production). @@ -2256,7 +2345,6 @@ pub struct Settings { #[serde(default)] pub consent: ConsentConfig, #[serde(default)] - #[validate(nested)] pub cache: CacheSettings, #[serde(default)] pub proxy: Proxy, @@ -3124,22 +3212,155 @@ mod tests { crate_test_settings_str() ); let settings = Settings::from_toml(&toml_str).expect("should parse cache asset rule"); + let expected_policy = CachePolicy::public_immutable(Duration::from_secs(31_536_000)); + + for path in [ + "/assets/app.0123abcd.js", + "/assets/index-DA15JTLU.js", + "/assets/index-BsELY24f.js", + "/assets/app-VRTVD5R5.js", + "/assets/app-VCMCQCKZ.js", + ] { + assert_eq!( + settings + .asset_cache_policy_for_path(path) + .expect("should evaluate cache rules"), + Some(expected_policy), + "supported fingerprint should match asset rule for {path}" + ); + } + + for path in ["/assets/app.js", "/assets/deadbeef.js"] { + assert!( + settings + .asset_cache_policy_for_path(path) + .expect("should evaluate cache rules") + .is_none(), + "non-fingerprinted filename should not match asset rule for {path}" + ); + } + } + + #[test] + fn filename_fingerprint_gate_supports_conservative_bundler_suffixes() { + for (path, expected) in [ + ("/assets/index-DA15JTLU.js", true), + ("/assets/index-BsELY24f.js", true), + ("/assets/index-aB_cD-12.js", true), + ("/assets/app-VRTVD5R5.js", true), + ("/assets/app-VCMCQCKZ.js", true), + ("/assets/app.a1B2c3D4.js", true), + ("/assets/main.a1b2c3d4e5f6.js", true), + ("/assets/index.8f3a2b1c.js", true), + ("/assets/app.deadbeef.js", true), + ("/assets/deadbeef.js", false), + ("/assets/VCMCQCKZ.js", false), + ("/assets/app.js", false), + ("/assets/app-manifest.js", false), + ("/assets/app-release2.js", false), + ("/assets/app.20260714.js", false), + ("/assets/app-abc123.js", false), + ("/assets/deadbeef/app.js", false), + ] { + assert_eq!( + filename_contains_fingerprint(path), + expected, + "fingerprint result should match for {path}" + ); + } + } + #[test] + fn disabled_cache_asset_rules_defer_matcher_and_policy_validation() { + let toml_str = format!( + r#"{} + + [[cache.asset_rules]] + id = "disabled-invalid-regex" + enabled = false + path_regex = "[" + + [[cache.asset_rules]] + id = "disabled-placeholder" + enabled = false + + [[cache.asset_rules]] + id = "disabled-unsafe-immutable" + enabled = false + path_prefix = "/assets/" + immutable = true + "#, + crate_test_settings_str() + ); + + let settings = + Settings::from_toml(&toml_str).expect("should defer disabled rule validation"); assert!( settings - .asset_cache_policy_for_path("/assets/app.js") - .expect("should evaluate cache rules") + .asset_cache_policy_for_path("/assets/app-DA15JTLU.js") + .expect("should evaluate disabled cache rules") .is_none(), - "broad allowlist should not match non-fingerprinted files when hash is required" + "disabled rules should never match" ); - assert_eq!( - settings - .asset_cache_policy_for_path("/assets/app.0123abcd.js") - .expect("should evaluate cache rules"), - Some(CachePolicy::public_immutable(Duration::from_secs( - 31_536_000 - ))), - "fingerprinted asset should match the allowlist" + } + + #[test] + fn cache_asset_rule_policy_validation_rejects_unsafe_config() { + let missing_ttl = format!( + r#"{} + + [[cache.asset_rules]] + id = "missing-ttl" + enabled = true + path_prefix = "/assets/" + "#, + crate_test_settings_str() + ); + let missing_ttl_err = + Settings::from_toml(&missing_ttl).expect_err("should reject rule without a TTL"); + assert!( + format!("{missing_ttl_err:?}").contains("browser_ttl_seconds or edge_ttl_seconds"), + "should explain missing TTL: {missing_ttl_err:?}" + ); + + let immutable_without_fingerprint = format!( + r#"{} + + [[cache.asset_rules]] + id = "unsafe-immutable" + enabled = true + path_prefix = "/assets/" + browser_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + ); + let fingerprint_err = Settings::from_toml(&immutable_without_fingerprint) + .expect_err("should reject immutable rule without fingerprint requirement"); + assert!( + format!("{fingerprint_err:?}").contains("requires_hash_in_filename"), + "should explain immutable fingerprint requirement: {fingerprint_err:?}" + ); + + let immutable_without_browser_ttl = format!( + r#"{} + + [[cache.asset_rules]] + id = "immutable-without-browser-ttl" + enabled = true + path_prefix = "/assets/" + requires_hash_in_filename = true + browser_ttl_seconds = 0 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + ); + let browser_ttl_err = Settings::from_toml(&immutable_without_browser_ttl) + .expect_err("should reject immutable rule without positive browser TTL"); + assert!( + format!("{browser_ttl_err:?}").contains("positive browser_ttl_seconds"), + "should explain immutable browser TTL requirement: {browser_ttl_err:?}" ); } @@ -3201,6 +3422,23 @@ mod tests { format!("{shape_err:?}").contains("exactly one matcher"), "should explain invalid matcher shape: {shape_err:?}" ); + + let missing_matcher = format!( + r#"{} + + [[cache.asset_rules]] + id = "missing-matcher" + enabled = true + browser_ttl_seconds = 60 + "#, + crate_test_settings_str() + ); + let missing_matcher_err = + Settings::from_toml(&missing_matcher).expect_err("should reject missing matcher"); + assert!( + format!("{missing_matcher_err:?}").contains("exactly one matcher"), + "should explain missing matcher: {missing_matcher_err:?}" + ); } #[test] diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 258a9a4bc..e085a23c1 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1042,29 +1042,48 @@ content-addressed or otherwise safe for the configured TTL. ### `[[cache.asset_rules]]` Rules are evaluated in file order; the first enabled matching rule wins. -Disabled rules are ignored, which lets you keep framework presets documented in -config without enabling them for every publisher. - -| Field | Type | Required | Description | -| -------------------------------- | ------------- | -------- | ----------------------------------------------------------------- | -| `id` | String | Yes | Unique operator-facing rule identifier | -| `enabled` | Boolean | No | Whether the rule participates in matching (default `false`) | -| `preset` | String | Matcher | Built-in preset such as `nextjs-static` | -| `path_prefix` | String | Matcher | Request path prefix | -| `path_glob` | String | Matcher | Single glob matched against the request path | -| `path_globs` | Array[String] | Matcher | Multiple globs matched against the request path | -| `path_regex` | String | Matcher | Regex matched against the request path | -| `extensions` | Array[String] | Matcher | Case-insensitive file extensions | -| `requires_hash_in_filename` | Boolean | No | Require an 8+ hex token in the final path segment before matching | -| `visibility` | String | No | `public` or `private` (default `public`) | -| `browser_ttl_seconds` | Integer | No | Browser `max-age` | -| `edge_ttl_seconds` | Integer | No | Edge/shared-cache TTL | -| `stale_while_revalidate_seconds` | Integer | No | Optional `stale-while-revalidate` | -| `stale_if_error_seconds` | Integer | No | Optional `stale-if-error` | -| `immutable` | Boolean | No | Add `immutable` when browser TTL is positive | - -Exactly one matcher must be configured per rule. `path_glob` and `path_globs` -are mutually exclusive. +Disabled rules never match, and their matcher and policy validation is deferred +until they are enabled. Rule IDs are always normalized and must remain nonempty +and unique, including for disabled placeholders. + +| Field | Type | Required | Description | +| -------------------------------- | ------------- | -------- | --------------------------------------------------------------- | +| `id` | String | Yes | Unique operator-facing rule identifier | +| `enabled` | Boolean | No | Whether the rule participates in matching (default `false`) | +| `preset` | String | Matcher | Built-in preset such as `nextjs-static` | +| `path_prefix` | String | Matcher | Request path prefix | +| `path_glob` | String | Matcher | Single glob matched against the request path | +| `path_globs` | Array[String] | Matcher | Multiple globs matched against the request path | +| `path_regex` | String | Matcher | Regex matched against the request path | +| `extensions` | Array[String] | Matcher | Case-insensitive file extensions | +| `requires_hash_in_filename` | Boolean | No | Require a supported bundler fingerprint suffix before matching | +| `visibility` | String | No | `public` or `private` (default `public`) | +| `browser_ttl_seconds` | Integer | Policy | Browser `max-age`; must be positive when `immutable = true` | +| `edge_ttl_seconds` | Integer | Policy | TTL emitted through the runtime-specific shared-cache directive | +| `stale_while_revalidate_seconds` | Integer | No | Optional `stale-while-revalidate` | +| `stale_if_error_seconds` | Integer | No | Optional `stale-if-error` | +| `immutable` | Boolean | No | Add `immutable` for a validated content-addressed rule | + +An enabled rule must configure exactly one matcher and at least one of +`browser_ttl_seconds` or `edge_ttl_seconds`. `path_glob` and `path_globs` are +mutually exclusive. `immutable = true` additionally requires a positive browser +TTL and either the content-addressed `nextjs-static` preset or +`requires_hash_in_filename = true`. + +The filename fingerprint check is intentionally conservative. It examines the +suffix immediately before the final extension, requires a nonempty filename +prefix separated by `.`, `-`, `_`, or `~`, and recognizes: + +- hexadecimal suffixes of at least eight characters containing a letter; +- eight-character esbuild-style uppercase Base32 suffixes; +- eight-character Vite/Base64URL-style suffixes with a mixed character class. + +For example, `app.0123abcd.js`, `app-VRTVD5R5.js`, and +`index-DA15JTLU.js` match, while `app.js`, `deadbeef.js`, and +`app.20260714.js` do not. This heuristic is not proof of content addressing; +confirm the publisher's bundler output before enabling a long immutable TTL. A +base rule that matches while this fingerprint check fails emits a debug log with +the rule ID and rejected path. **Next.js preset example** (disabled until the publisher confirms `/_next/static/` is content-addressed): @@ -1080,12 +1099,13 @@ edge_ttl_seconds = 31536000 immutable = true ``` -**Publisher allowlist example**: +**Publisher allowlist example** (enable only after verifying the filename +convention): ```toml [[cache.asset_rules]] id = "publisher-fingerprinted-assets" -enabled = true +enabled = false path_globs = [ "/assets/**/*.js", "/assets/**/*.css", @@ -1100,9 +1120,28 @@ immutable = true ``` If `[cache]` is omitted or no enabled rule matches, Trusted Server preserves the -origin cache policy for publisher-origin assets. TS-owned validated hash URLs, -such as `/static/tsjs=...js?v=`, use their built-in cache policy and do -not require an asset rule. +origin cache policy for publisher-origin assets. On the publisher pass-through +path, an origin `private` or `no-store` directive vetoes a matching rule. Other +origin cache directives, including `no-cache`, are replaced by the configured +policy. `Vary` is preserved, so do not assign a public immutable rule to paths +that vary by cookies or other user-specific request state. + +On a configured Fastly asset-rehost route, a matching rule is authoritative +over the third-party origin's cache defaults, including `no-store`, because +Trusted Server owns the rehosted copy. A later Trusted Server or operator-applied +`private` or `no-store` directive still vetoes public policy reapplication and +removes shared-cache headers. + +TS-owned validated hash URLs such as `/static/tsjs=...js?v=` use their +built-in cache policy and do not require an asset rule. Shared-cache keys for +`/static/tsjs=` must preserve `v`; otherwise a matching immutable response can +collide with the missing or mismatched version's short-TTL response. + +`edge_ttl_seconds` only emits the selected runtime's shared-cache directive. The +runtime or service must also enable and consume that directive. The checked-in +Cloudflare manifests enable Workers Cache. Fastly synthetic and final egress +responses still require explicit runtime cache integration, tracked in +[#908](https://github.com/IABTechLab/trusted-server/issues/908). ## Integration Configurations diff --git a/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md b/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md index f156aa768..ce2c37956 100644 --- a/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md +++ b/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md @@ -44,11 +44,14 @@ into the initial cache-header PRs. - TS-owned Prebid delivery is covered by deferred TSJS module URLs. Publisher Prebid script URLs neutralized by TS are compatibility shims at stable URLs and must use `no-store` or a very short TTL, not a year-long immutable policy. -- Rehosted assets are TS-owned copies once TS rewrites/hosts them. They should - use explicit normalized policies, with immutable only for TS-fingerprinted - rehosted URLs. -- Fastly and Cloudflare are the MVP runtime targets. Akamai mapping is deferred - until Akamai is on the roadmap. +- Fastly rehosted assets are TS-owned copies once TS rewrites/hosts them. A + matching rehost rule is authoritative over third-party origin cache defaults. Use + immutable only for TS-fingerprinted rehosted URLs, and preserve any later + TS/operator `private` or `no-store` decision as the final veto. +- Fastly and Cloudflare are the MVP runtime targets. This slice emits their + runtime-specific directives; actual Fastly storage integration and cache-key + verification are tracked in [#908](https://github.com/IABTechLab/trusted-server/issues/908). + Akamai mapping is deferred until Akamai is on the roadmap. - Dynamic HTML/RSC/API caching, dynamic `Vary`/cache-key normalization, origin-template caching, transformed-template caching, true publisher-origin streaming, parser-context bid splice, EdgeZero streaming parity, and SSAT HTML @@ -88,11 +91,13 @@ into the initial cache-header PRs. implemented through the shared rule engine and can be disabled/overridden. - Arbitrary publisher-origin assets remain origin-controlled unless matched by an enabled preset or publisher allowlist. -- TS-owned rehosted assets have explicit normalized policies instead of blindly - passing through third-party defaults. +- Fastly TS-owned rehosted assets have explicit normalized policies instead of + blindly passing through third-party defaults. - MVP adapters emit the correct edge-cache header from shared policy: Fastly `Surrogate-Control`, Cloudflare `CDN-Cache-Control` / - `Cloudflare-CDN-Cache-Control`, or portable `s-maxage` fallback. + `Cloudflare-CDN-Cache-Control`, or portable `s-maxage` fallback. Header + emission is complete; runtime storage and cache-key verification remain in + #908. - Deferred features are documented as deferred and are not accidentally implemented as hard-coded Next.js/dynamic-cache behavior. - Tests and target-matched checks pass for touched crates/adapters. @@ -244,7 +249,8 @@ upgrades to publisher-origin assets. Make the shared policy output explicit per runtime before wiring the rule engine into more routes. This prevents new code from copying the current core -Fastly-specific `Surrogate-Control` behavior. +Fastly-specific `Surrogate-Control` behavior. This phase covers directive +rendering only; runtime storage is tracked in #908. #### Code changes @@ -283,10 +289,14 @@ assets, using the runtime edge-header mapping from PR 5. - `OriginControlled` - `NoStorePrivate` - `Normalized(CachePolicy)` from a matched enabled rule. -- Apply normalized policy after route finalization but before final response - privacy hardening. +- Apply normalized policy at the asset handler, then reapply its runtime edge + directive after route finalization only when the finalized response is still + cacheable. Final `private` or `no-store` directives veto reapplication and + remove edge-cache headers. - Preserve existing no-store/private handling for errors, signed failures, or - responses that set cookies/security headers. + responses that set cookies/security headers. A matched TS-owned rehost rule + intentionally replaces the third-party origin's cache defaults before this + final privacy veto. - Ensure operator `response_headers` cannot weaken protected private/no-store decisions. - For TS-owned rehosted copies: diff --git a/docs/superpowers/specs/2026-07-06-cache-control-header-design.md b/docs/superpowers/specs/2026-07-06-cache-control-header-design.md index 8dae6aca2..1924381f1 100644 --- a/docs/superpowers/specs/2026-07-06-cache-control-header-design.md +++ b/docs/superpowers/specs/2026-07-06-cache-control-header-design.md @@ -42,7 +42,7 @@ Origin ──▶ TS edge/shared cache ──▶ Browser cache - Portable fallback: `s-maxage` inside `Cache-Control` - **Browser cache:** controlled by `max-age` and related `Cache-Control` directives. -A single `max-age` cannot express “hold at the edge for a year, but revalidate in the browser daily” or the reverse. TS should model these tiers separately and let adapters render the appropriate headers. +A single `max-age` cannot express “hold at the edge for a year, but revalidate in the browser daily” or the reverse. TS should model these tiers separately and let adapters render the appropriate headers. Header rendering alone does not enable a runtime cache: Cloudflare Workers Cache must be enabled, and Fastly synthetic/final egress responses require explicit cache integration tracked in [#908](https://github.com/IABTechLab/trusted-server/issues/908). ## Policy model @@ -63,18 +63,18 @@ Rules should be configurable. Built-in framework presets, such as Next.js `/_nex ## Target behavior by response class -| Response class | Target policy | Notes | -| --------------------------------------------------------------- | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | -| TSJS with matching `?v=` | `Cache-Control: public, max-age=31536000, immutable` plus runtime edge header | The serving path must validate that `v` matches the bytes served. | -| TSJS missing/mismatched `?v=` | Short TTL or redirect to canonical hashed URL | Do not mark immutable. | -| TSJS deferred modules, including Prebid | Same as TSJS hash-matching policy | Example: `/static/tsjs=tsjs-prebid.min.js?v=`. | -| Publisher Prebid URL neutralized by TS | `no-store` or very short TTL | The empty compatibility shim is config-dependent and served at a stable publisher URL. Do not cache it for a year. | -| Enabled framework preset static, e.g. Next.js `/_next/static/*` | `Cache-Control: public, max-age=31536000, immutable` | Applied through configurable preset/allowlist rules. | -| TS-fingerprinted rehosted asset | `Cache-Control: public, max-age=31536000, immutable` | Safe only when TS owns the fingerprinted URL. | -| Stable TS-owned/rehosted URL | Conservative short browser TTL, optional longer edge TTL | Do not use `immutable`. | -| Arbitrary publisher-origin CSS/JS/images | Origin-controlled by default | TS may upgrade only via enabled framework preset or publisher allowlist. | -| SSAT-assembled ad-stack HTML | `Cache-Control: private, max-age=0`; strip runtime edge-cache headers | Must never enter shared cache because it can contain per-user slot/bid data. | -| Dynamic HTML/RSC/API | Origin-controlled in this slice | Future dynamic caching belongs to #859. | +| Response class | Target policy | Notes | +| --------------------------------------------------------------- | ----------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| TSJS with matching `?v=` | `Cache-Control: public, max-age=31536000, immutable` plus runtime edge header | The serving path must validate that `v` matches the bytes served. | +| TSJS missing/mismatched `?v=` | Short TTL or redirect to canonical hashed URL | Do not mark immutable. | +| TSJS deferred modules, including Prebid | Same as TSJS hash-matching policy | Example: `/static/tsjs=tsjs-prebid.min.js?v=`. | +| Publisher Prebid URL neutralized by TS | `no-store` or very short TTL | The empty compatibility shim is config-dependent and served at a stable publisher URL. Do not cache it for a year. | +| Enabled framework preset static, e.g. Next.js `/_next/static/*` | `Cache-Control: public, max-age=31536000, immutable` | Applied through configurable preset/allowlist rules. | +| Fastly TS-fingerprinted rehosted asset | `Cache-Control: public, max-age=31536000, immutable` | Safe only when TS owns the fingerprinted URL. A matched rehost rule is authoritative over third-party origin cache defaults. | +| Stable Fastly TS-owned/rehosted URL | Conservative short browser TTL, optional longer edge TTL | Do not use `immutable`; later TS/operator `private` or `no-store` finalization remains a veto. | +| Arbitrary publisher-origin CSS/JS/images | Origin-controlled by default | TS may upgrade only via enabled framework preset or publisher allowlist. | +| SSAT-assembled ad-stack HTML | `Cache-Control: private, max-age=0`; strip runtime edge-cache headers | Must never enter shared cache because it can contain per-user slot/bid data. | +| Dynamic HTML/RSC/API | Origin-controlled in this slice | Future dynamic caching belongs to #859. | ## TSJS-specific requirements @@ -126,20 +126,20 @@ Adapters should render the shared policy as follows: | Cloudflare | `CDN-Cache-Control` / `Cloudflare-CDN-Cache-Control` | | Portable fallback | `s-maxage` in `Cache-Control` | -Akamai mapping is deferred until Akamai is on the roadmap. +These mappings define emitted directives, not storage by themselves. The runtime must enable or implement the corresponding shared-cache mechanism. Akamai mapping is deferred until Akamai is on the roadmap. ## Acceptance criteria -- [ ] Cache policy is represented as structured fields, not hard-coded header strings. -- [ ] Built-in framework presets, including a disableable/overrideable Next.js `/_next/static/*` rule, are implemented through the shared cache-policy rule engine. -- [ ] TSJS hash-matching requests for unified and deferred modules emit `public, max-age=31536000, immutable` plus the runtime edge header. -- [ ] TSJS missing/mismatched hash requests do not get immutable caching. -- [ ] TSJS hash generation is build-time or cached enough that HTML injection does not re-concatenate/re-hash the bundle per pageview. -- [ ] Runtime cache-key configuration preserves the `v` query parameter for `/static/tsjs=`. -- [ ] Neutralized publisher Prebid shim responses use `no-store` or a short TTL, not a year-long policy. -- [ ] Arbitrary publisher-origin assets remain origin-controlled unless covered by an enabled framework preset or publisher allowlist. -- [ ] TS-owned rehosted assets have explicit normalized cache policy; immutable is used only for TS-fingerprinted rehosted URLs. -- [ ] SSAT-assembled ad-stack HTML continues to emit `private, max-age=0` and strips all runtime edge-cache headers. -- [ ] Fastly and Cloudflare adapters emit the correct edge-cache header from the shared policy, with portable `s-maxage` fallback where needed. -- [ ] Dynamic HTML/RSC/API Vary/cache-key normalization is not hard-coded in this PR and is deferred to #859. -- [ ] SSAT streaming fixes and compression offload are not included in this PR and remain tracked by #857 and #858. +- [x] Cache policy is represented as structured fields, not hard-coded header strings. +- [x] Built-in framework presets, including a disableable/overrideable Next.js `/_next/static/*` rule, are implemented through the shared cache-policy rule engine. +- [x] TSJS hash-matching requests for unified and deferred modules emit `public, max-age=31536000, immutable` plus the runtime edge header. +- [x] TSJS missing/mismatched hash requests do not get immutable caching. +- [x] TSJS hash generation is build-time or cached enough that HTML injection does not re-concatenate/re-hash the bundle per pageview. +- [ ] Runtime cache-key configuration preserves the `v` query parameter for `/static/tsjs=`. Runtime verification is tracked in #908. +- [x] Neutralized publisher Prebid shim responses use `no-store` or a short TTL, not a year-long policy. +- [x] Arbitrary publisher-origin assets remain origin-controlled unless covered by an enabled framework preset or publisher allowlist. +- [x] Fastly TS-owned rehosted assets have explicit normalized cache policy; immutable is used only for TS-fingerprinted rehosted URLs. +- [x] SSAT-assembled ad-stack HTML continues to emit `private, max-age=0` and strips all runtime edge-cache headers. +- [x] Fastly and Cloudflare adapters emit the correct edge-cache header from the shared policy, with portable `s-maxage` fallback where needed. Actual shared-cache storage remains tracked in #908. +- [x] Dynamic HTML/RSC/API Vary/cache-key normalization is not hard-coded in this PR and is deferred to #859. +- [x] SSAT streaming fixes and compression offload are not included in this PR and remain tracked by #857 and #858. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 2b46336ab..ca566fcb6 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -116,8 +116,10 @@ enabled = false # Required for integrations.prebid.external_bundle_url and first-party proxy redirects. # allowed_domains = ["ads.example.com", "assets.example.com", "*.cdn.example.com"] -# Static/rehosted asset cache policies are operator-controlled. Keep framework -# presets disabled unless the matched publisher paths are known content-addressed. +# Static/rehosted asset cache policies are operator-controlled. Disabled rules +# do not match, and matcher/policy validation is deferred until they are enabled; +# IDs must still be nonempty and unique. Keep rules disabled unless the matched +# publisher paths are known content-addressed. # [[cache.asset_rules]] # id = "nextjs-static" # enabled = false @@ -131,6 +133,8 @@ enabled = false # id = "publisher-fingerprinted-assets" # enabled = false # path_globs = ["/assets/**/*.js", "/assets/**/*.css", "/assets/**/*.png", "/assets/**/*.webp"] +# Immutable custom rules require a supported name-delimited bundler fingerprint +# immediately before the extension, for example app.0123abcd.js or app-VRTVD5R5.js. # requires_hash_in_filename = true # visibility = "public" # browser_ttl_seconds = 31536000 From 2de213d152682425c4c615cb70ae46c74e0abd11 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 17 Jul 2026 10:12:00 -0500 Subject: [PATCH 282/395] Fix cache policy clippy warnings --- .../trusted-server-core/src/cache_policy.rs | 25 +++++++++---------- crates/trusted-server-core/src/proxy.rs | 8 +++--- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/crates/trusted-server-core/src/cache_policy.rs b/crates/trusted-server-core/src/cache_policy.rs index 5b2b09a3e..39bac467c 100644 --- a/crates/trusted-server-core/src/cache_policy.rs +++ b/crates/trusted-server-core/src/cache_policy.rs @@ -159,13 +159,12 @@ impl CachePolicy { directives.push(format!("max-age={}", ttl.as_secs())); } - if edge_header == EdgeCacheHeader::SMaxageFallback { - if let Some(ttl) = self + if edge_header == EdgeCacheHeader::SMaxageFallback + && let Some(ttl) = self .edge_ttl .filter(|_| self.visibility == CacheVisibility::Public) - { - directives.push(format!("s-maxage={}", ttl.as_secs())); - } + { + directives.push(format!("s-maxage={}", ttl.as_secs())); } if let Some(ttl) = self.stale_while_revalidate { @@ -226,14 +225,14 @@ impl CachePolicy { ); remove_edge_cache_headers(headers); - if let Some(header_name) = edge_header.header_name() { - if let Some(value) = self.edge_header_value(edge_header) { - headers.insert( - header_name, - HeaderValue::from_str(&value) - .expect("should render a valid edge cache-control header"), - ); - } + if let Some(header_name) = edge_header.header_name() + && let Some(value) = self.edge_header_value(edge_header) + { + headers.insert( + header_name, + HeaderValue::from_str(&value) + .expect("should render a valid edge cache-control header"), + ); } } } diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index b51cd75f6..a2ff43fca 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -1228,10 +1228,10 @@ pub async fn handle_asset_proxy_request( strip_asset_proxy_response_headers(response.response_mut()); let status = response.response().status(); - if status.is_success() || status == StatusCode::NOT_MODIFIED { - if let Some(policy) = settings.asset_cache_policy_for_path(incoming_path)? { - response.apply_normalized_cache_policy(policy); - } + if (status.is_success() || status == StatusCode::NOT_MODIFIED) + && let Some(policy) = settings.asset_cache_policy_for_path(incoming_path)? + { + response.apply_normalized_cache_policy(policy); } Ok(response) From d6920d45d45a23737b8c657568659d862337f49c Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 29 Jul 2026 10:46:10 -0500 Subject: [PATCH 283/395] Fix publisher test cache policy argument --- crates/trusted-server-core/src/publisher.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 674d3d718..9cd70290a 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -9442,6 +9442,7 @@ mod tests { registry: None, }, req, + EdgeCacheHeader::SurrogateControl, ) .await .expect("should proxy publisher request"); From 3b9063f72d3b210ea4c93b829b9e4f53452b0028 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 29 Jul 2026 13:51:09 -0500 Subject: [PATCH 284/395] Preserve integration configuration values on rebase --- crates/trusted-server-core/src/settings.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index effbf613f..870014c2d 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2429,7 +2429,6 @@ impl Settings { mut settings: Self, validation_label: &str, ) -> Result> { - settings.integrations.normalize(); settings.cache.normalize(); settings.proxy.normalize(); settings.image_optimizer.normalize(); From f4169fb6e277c534cc3fb05c9d8b5aee0ab7463b Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 30 Jul 2026 12:57:39 -0500 Subject: [PATCH 285/395] Harden configurable asset cache rules --- crates/trusted-server-core/src/proxy.rs | 4 +- crates/trusted-server-core/src/publisher.rs | 15 +- crates/trusted-server-core/src/settings.rs | 316 ++++++++++++------ crates/trusted-server-js/src/bundle.rs | 7 +- docs/guide/configuration.md | 45 +-- ...ache-control-header-implementation-plan.md | 2 +- trusted-server.example.toml | 6 +- 7 files changed, 262 insertions(+), 133 deletions(-) diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index a2ff43fca..2ad8091fd 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -4353,7 +4353,7 @@ mod tests { id = "fingerprinted-assets" enabled = true path_globs = ["/assets/**/*.js"] - requires_hash_in_filename = true + fingerprint_style = "hex" visibility = "public" browser_ttl_seconds = 31536000 edge_ttl_seconds = 31536000 @@ -4463,7 +4463,7 @@ mod tests { id = "fingerprinted-assets" enabled = true path_globs = ["/assets/**/*.js"] - requires_hash_in_filename = true + fingerprint_style = "hex" visibility = "public" browser_ttl_seconds = 31536000 edge_ttl_seconds = 31536000 diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 9cd70290a..df57a3450 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1081,11 +1081,12 @@ fn response_cache_control_is_private_or_no_store(response: &Response) fn apply_publisher_asset_cache_policy( settings: &Settings, path: &str, - cache_rule_method: bool, + method: &Method, edge_header: EdgeCacheHeader, response: &mut Response, ) -> Result<(), Report> { - if !cache_rule_method || response_cache_control_is_private_or_no_store(response) { + let is_cacheable_method = *method == Method::GET || *method == Method::HEAD; + if !is_cacheable_method || response_cache_control_is_private_or_no_store(response) { return Ok(()); } @@ -2659,8 +2660,8 @@ pub async fn handle_publisher_request( log::debug!("Proxying request to configured publisher backend"); let request_path = req.uri().path().to_string(); - let is_get = req.method() == http::Method::GET; - let cache_rule_method = req.method() == Method::GET || req.method() == Method::HEAD; + let request_method = req.method().clone(); + let is_get = request_method == Method::GET; let is_prefetch = is_prefetch_request(&req); let is_bot = is_bot_user_agent(&req); @@ -2944,7 +2945,7 @@ pub async fn handle_publisher_request( apply_publisher_asset_cache_policy( settings, &request_path, - cache_rule_method, + &request_method, edge_header, &mut response, )?; @@ -4432,7 +4433,7 @@ mod tests { id = "publisher-fingerprinted-assets" enabled = true path_globs = ["/assets/**/*.png"] - requires_hash_in_filename = true + fingerprint_style = "hex" visibility = "public" browser_ttl_seconds = 31536000 edge_ttl_seconds = 31536000 @@ -4523,7 +4524,7 @@ mod tests { id = "publisher-fingerprinted-assets" enabled = true path_globs = ["/assets/**/*.png"] - requires_hash_in_filename = true + fingerprint_style = "hex" visibility = "public" browser_ttl_seconds = 31536000 edge_ttl_seconds = 31536000 diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 870014c2d..da61dd84a 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1,7 +1,7 @@ #[cfg(test)] use config::{Config, Environment, File, FileFormat}; use error_stack::{Report, ResultExt}; -use glob::Pattern; +use glob::{MatchOptions, Pattern}; use regex::Regex; use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; @@ -1957,9 +1957,9 @@ pub struct CacheAssetRule { /// File extensions matched against the request path, case-insensitively. #[serde(default)] pub extensions: Vec, - /// Require a supported bundler fingerprint suffix in the filename before matching. + /// Bundler fingerprint style required in the filename before matching. #[serde(default)] - pub requires_hash_in_filename: bool, + pub fingerprint_style: Option, /// Browser-facing cache visibility. #[serde(default)] pub visibility: CachePolicyVisibility, @@ -2083,10 +2083,10 @@ impl CacheAssetRule { let preset_is_content_addressed = matches!(self.preset, Some(CacheAssetPreset::NextJsStatic)); - if !preset_is_content_addressed && !self.requires_hash_in_filename { + if !preset_is_content_addressed && self.fingerprint_style.is_none() { return Err(Report::new(TrustedServerError::Configuration { message: format!( - "cache.asset_rules `{}` sets immutable without requires_hash_in_filename or a content-addressed preset", + "cache.asset_rules `{}` sets immutable without fingerprint_style or a content-addressed preset", self.id ), })); @@ -2119,16 +2119,16 @@ impl CacheAssetRule { } match self.compiled_globs.get_or_init(|| { - if let Some(glob) = self.path_glob.as_deref() { - Pattern::new(glob) - .map(|pattern| vec![pattern]) - .map_err(|err| err.to_string()) - } else { - self.path_globs - .iter() - .map(|pattern| Pattern::new(pattern).map_err(|err| err.to_string())) - .collect() + let mut compiled = Vec::new(); + let source_patterns = self + .path_glob + .iter() + .chain(self.path_globs.iter()) + .map(String::as_str); + for pattern in source_patterns { + compile_cache_asset_glob_patterns(pattern, &mut compiled)?; } + Ok(compiled) }) { Ok(patterns) => Ok(Some(patterns.as_slice())), Err(message) => Err(Report::new(TrustedServerError::Configuration { @@ -2145,9 +2145,11 @@ impl CacheAssetRule { return Ok(false); } - if self.requires_hash_in_filename && !filename_contains_fingerprint(path) { + if let Some(style) = self.fingerprint_style + && !filename_contains_fingerprint(path, style) + { log::debug!( - "cache asset rule `{}` rejects path `{path}` because the filename has no supported fingerprint", + "cache asset rule `{}` rejects path `{path}` because the filename has no {style:?} fingerprint", self.id ); return Ok(false); @@ -2164,7 +2166,9 @@ impl CacheAssetRule { return Ok(path.starts_with(prefix)); } if let Some(patterns) = self.compiled_globs()? { - return Ok(patterns.iter().any(|pattern| pattern.matches(path))); + return Ok(patterns + .iter() + .any(|pattern| pattern.matches_with(path, CACHE_ASSET_GLOB_MATCH_OPTIONS))); } if let Some(regex) = self.compiled_regex()? { return Ok(regex.is_match(path)); @@ -2191,6 +2195,30 @@ impl CacheAssetRule { } } +const CACHE_ASSET_GLOB_MATCH_OPTIONS: MatchOptions = MatchOptions { + case_sensitive: true, + require_literal_separator: true, + require_literal_leading_dot: false, +}; + +fn compile_cache_asset_glob_patterns( + pattern: &str, + compiled: &mut Vec, +) -> Result<(), String> { + compiled.push(Pattern::new(pattern).map_err(|err| err.to_string())?); + + if let Some(optional_recursive_start) = pattern.find("**/") { + let without_recursive_segment = format!( + "{}{}", + &pattern[..optional_recursive_start], + &pattern[optional_recursive_start + "**/".len()..] + ); + compile_cache_asset_glob_patterns(&without_recursive_segment, compiled)?; + } + + Ok(()) +} + /// Built-in cache-rule presets that operators can enable explicitly. #[derive(Debug, Clone, Copy, Deserialize, Serialize, Eq, PartialEq)] #[serde(rename_all = "kebab-case")] @@ -2234,7 +2262,48 @@ fn path_extension(path: &str) -> Option { (!extension.is_empty()).then(|| extension.to_ascii_lowercase()) } -fn filename_contains_fingerprint(path: &str) -> bool { +/// Operator-selected filename fingerprint convention for an immutable custom rule. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub enum CacheAssetFingerprintStyle { + /// A hexadecimal suffix, such as `app.0123abcd.js`. + Hex, + /// An eight-character uppercase Base32 suffix, such as `app-VRTVD5R5.js`. + EsbuildBase32, + /// An eight-character `Base64URL` suffix, such as `index-BsELY24f.js`. + ViteBase64Url, +} + +impl CacheAssetFingerprintStyle { + fn matches_candidate(self, candidate: &str) -> bool { + match self { + Self::Hex => { + candidate.len() >= 8 + && candidate.chars().all(|ch| ch.is_ascii_hexdigit()) + && candidate.chars().any(|ch| ch.is_ascii_alphabetic()) + } + Self::EsbuildBase32 => { + candidate.len() == 8 + && candidate + .chars() + .all(|ch| ch.is_ascii_uppercase() || matches!(ch, '2'..='7')) + && candidate.chars().any(|ch| ch.is_ascii_alphabetic()) + } + Self::ViteBase64Url => { + candidate.len() == 8 + && candidate + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_')) + && candidate.chars().any(|ch| ch.is_ascii_uppercase()) + && candidate.chars().any(|ch| { + ch.is_ascii_lowercase() || ch.is_ascii_digit() || matches!(ch, '-' | '_') + }) + } + } + } +} + +fn filename_contains_fingerprint(path: &str, style: CacheAssetFingerprintStyle) -> bool { let filename = path.rsplit('/').next().unwrap_or(path); let Some((stem, extension)) = filename.rsplit_once('.') else { return false; @@ -2249,31 +2318,10 @@ fn filename_contains_fingerprint(path: &str) -> bool { let candidate_start = separator_index + separator.len_utf8(); let prefix = &stem[..separator_index]; let candidate = &stem[candidate_start..]; - !prefix.is_empty() && fingerprint_candidate_is_supported(candidate) + !prefix.is_empty() && style.matches_candidate(candidate) }) } -fn fingerprint_candidate_is_supported(candidate: &str) -> bool { - let is_hex = candidate.len() >= 8 - && candidate.chars().all(|ch| ch.is_ascii_hexdigit()) - && candidate.chars().any(|ch| ch.is_ascii_alphabetic()); - let is_esbuild_base32 = candidate.len() == 8 - && candidate - .chars() - .all(|ch| ch.is_ascii_uppercase() || matches!(ch, '2'..='7')) - && candidate.chars().any(|ch| ch.is_ascii_alphabetic()); - let is_vite_base64url = candidate.len() == 8 - && candidate - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_')) - && candidate.chars().any(|ch| ch.is_ascii_uppercase()) - && candidate - .chars() - .any(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || matches!(ch, '-' | '_')); - - is_hex || is_esbuild_base32 || is_vite_base64url -} - /// Debug-only features. All flags default to `false` (off in production). #[derive(Debug, Default, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -3194,77 +3242,149 @@ mod tests { } #[test] - fn cache_asset_rule_requires_hash_in_filename_when_configured() { - let toml_str = format!( - r#"{} - - [[cache.asset_rules]] - id = "publisher-assets" - enabled = true - path_globs = ["/assets/**/*.js"] - requires_hash_in_filename = true - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - "#, - crate_test_settings_str() - ); - let settings = Settings::from_toml(&toml_str).expect("should parse cache asset rule"); + fn cache_asset_rule_requires_selected_fingerprint_style() { let expected_policy = CachePolicy::public_immutable(Duration::from_secs(31_536_000)); - - for path in [ - "/assets/app.0123abcd.js", - "/assets/index-DA15JTLU.js", - "/assets/index-BsELY24f.js", - "/assets/app-VRTVD5R5.js", - "/assets/app-VCMCQCKZ.js", + for (style, matching_path, non_matching_path) in [ + ("hex", "/assets/app.0123abcd.js", "/assets/app-VRTVD5R5.js"), + ( + "esbuild-base32", + "/assets/app-VRTVD5R5.js", + "/assets/index-BsELY24f.js", + ), + ( + "vite-base64-url", + "/assets/index-BsELY24f.js", + "/assets/app.0123abcd.js", + ), ] { + let toml_str = format!( + r#"{} + + [[cache.asset_rules]] + id = "publisher-assets" + enabled = true + path_globs = ["/assets/**/*.js"] + fingerprint_style = "{style}" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse cache asset rule"); + assert_eq!( settings - .asset_cache_policy_for_path(path) + .asset_cache_policy_for_path(matching_path) .expect("should evaluate cache rules"), Some(expected_policy), - "supported fingerprint should match asset rule for {path}" + "{style} should match its configured fingerprint convention" ); - } - - for path in ["/assets/app.js", "/assets/deadbeef.js"] { assert!( settings - .asset_cache_policy_for_path(path) + .asset_cache_policy_for_path(non_matching_path) .expect("should evaluate cache rules") .is_none(), - "non-fingerprinted filename should not match asset rule for {path}" + "{style} should not fall through to another fingerprint convention" ); } } #[test] - fn filename_fingerprint_gate_supports_conservative_bundler_suffixes() { - for (path, expected) in [ - ("/assets/index-DA15JTLU.js", true), - ("/assets/index-BsELY24f.js", true), - ("/assets/index-aB_cD-12.js", true), - ("/assets/app-VRTVD5R5.js", true), - ("/assets/app-VCMCQCKZ.js", true), - ("/assets/app.a1B2c3D4.js", true), - ("/assets/main.a1b2c3d4e5f6.js", true), - ("/assets/index.8f3a2b1c.js", true), - ("/assets/app.deadbeef.js", true), - ("/assets/deadbeef.js", false), - ("/assets/VCMCQCKZ.js", false), - ("/assets/app.js", false), - ("/assets/app-manifest.js", false), - ("/assets/app-release2.js", false), - ("/assets/app.20260714.js", false), - ("/assets/app-abc123.js", false), - ("/assets/deadbeef/app.js", false), + fn filename_fingerprint_gate_matches_only_the_selected_style() { + for (style, path, expected) in [ + ( + CacheAssetFingerprintStyle::Hex, + "/assets/app.0123abcd.js", + true, + ), + ( + CacheAssetFingerprintStyle::Hex, + "/assets/hero-Portrait.jpg", + false, + ), + ( + CacheAssetFingerprintStyle::EsbuildBase32, + "/assets/app-VRTVD5R5.js", + true, + ), + ( + CacheAssetFingerprintStyle::EsbuildBase32, + "/assets/hero-Portrait.jpg", + false, + ), + ( + CacheAssetFingerprintStyle::ViteBase64Url, + "/assets/index-BsELY24f.js", + true, + ), + ( + CacheAssetFingerprintStyle::ViteBase64Url, + "/assets/hero-Portrait.jpg", + true, + ), + ( + CacheAssetFingerprintStyle::ViteBase64Url, + "/assets/app.js", + false, + ), + ( + CacheAssetFingerprintStyle::ViteBase64Url, + "/assets/deadbeef.js", + false, + ), ] { assert_eq!( - filename_contains_fingerprint(path), + filename_contains_fingerprint(path, style), expected, - "fingerprint result should match for {path}" + "{style:?} fingerprint result should match for {path}" + ); + } + } + + #[test] + fn cache_asset_rule_globs_respect_path_separators() { + let toml_str = format!( + r#"{} + + [[cache.asset_rules]] + id = "direct-assets" + enabled = true + path_glob = "/assets/*.js" + browser_ttl_seconds = 300 + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse cache asset rule"); + + assert!( + settings + .asset_cache_policy_for_path("/assets/app.js") + .expect("should evaluate direct asset rule") + .is_some(), + "single-star glob should match a direct child" + ); + for path in ["/assets/vendor/app.js", "/assets/app.JS"] { + assert!( + settings + .asset_cache_policy_for_path(path) + .expect("should evaluate direct asset rule") + .is_none(), + "single-star glob should not match {path}" + ); + } + + let recursive_toml = toml_str.replace("/assets/*.js", "/assets/**/*.js"); + let recursive_settings = + Settings::from_toml(&recursive_toml).expect("should parse recursive cache asset rule"); + for path in ["/assets/app.js", "/assets/vendor/app.js"] { + assert!( + recursive_settings + .asset_cache_policy_for_path(path) + .expect("should evaluate recursive asset rule") + .is_some(), + "double-star glob should match {path}" ); } } @@ -3322,7 +3442,7 @@ mod tests { "should explain missing TTL: {missing_ttl_err:?}" ); - let immutable_without_fingerprint = format!( + let immutable_without_fingerprint_style = format!( r#"{} [[cache.asset_rules]] @@ -3334,11 +3454,11 @@ mod tests { "#, crate_test_settings_str() ); - let fingerprint_err = Settings::from_toml(&immutable_without_fingerprint) - .expect_err("should reject immutable rule without fingerprint requirement"); + let fingerprint_style_err = Settings::from_toml(&immutable_without_fingerprint_style) + .expect_err("should reject immutable rule without a fingerprint style"); assert!( - format!("{fingerprint_err:?}").contains("requires_hash_in_filename"), - "should explain immutable fingerprint requirement: {fingerprint_err:?}" + format!("{fingerprint_style_err:?}").contains("fingerprint_style"), + "should explain immutable fingerprint-style requirement: {fingerprint_style_err:?}" ); let immutable_without_browser_ttl = format!( @@ -3348,7 +3468,7 @@ mod tests { id = "immutable-without-browser-ttl" enabled = true path_prefix = "/assets/" - requires_hash_in_filename = true + fingerprint_style = "hex" browser_ttl_seconds = 0 edge_ttl_seconds = 31536000 immutable = true diff --git a/crates/trusted-server-js/src/bundle.rs b/crates/trusted-server-js/src/bundle.rs index 83815654a..be5aa35cc 100644 --- a/crates/trusted-server-js/src/bundle.rs +++ b/crates/trusted-server-js/src/bundle.rs @@ -36,9 +36,10 @@ pub fn concatenate_modules(ids: &[&str]) -> String { /// SHA-256 hash of the concatenated modules, for cache-busting URLs. /// /// The hash is computed over the same byte sequence as [`concatenate_modules`] -/// without allocating that concatenated body. Results are cached by ordered -/// module ID list so HTML injection does not re-hash the full JS payload on -/// every page view. +/// without allocating that concatenated body. Results are memoized by ordered +/// module ID list for reused processes or isolates. Fastly creates a fresh Wasm +/// instance per request, but still benefits from hashing without materializing +/// the concatenated body. #[must_use] #[inline] pub fn concatenated_hash(ids: &[&str]) -> String { diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index e085a23c1..96389be78 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1056,7 +1056,7 @@ and unique, including for disabled placeholders. | `path_globs` | Array[String] | Matcher | Multiple globs matched against the request path | | `path_regex` | String | Matcher | Regex matched against the request path | | `extensions` | Array[String] | Matcher | Case-insensitive file extensions | -| `requires_hash_in_filename` | Boolean | No | Require a supported bundler fingerprint suffix before matching | +| `fingerprint_style` | String | No | Required bundler fingerprint convention before matching | | `visibility` | String | No | `public` or `private` (default `public`) | | `browser_ttl_seconds` | Integer | Policy | Browser `max-age`; must be positive when `immutable = true` | | `edge_ttl_seconds` | Integer | Policy | TTL emitted through the runtime-specific shared-cache directive | @@ -1067,23 +1067,30 @@ and unique, including for disabled placeholders. An enabled rule must configure exactly one matcher and at least one of `browser_ttl_seconds` or `edge_ttl_seconds`. `path_glob` and `path_globs` are mutually exclusive. `immutable = true` additionally requires a positive browser -TTL and either the content-addressed `nextjs-static` preset or -`requires_hash_in_filename = true`. - -The filename fingerprint check is intentionally conservative. It examines the -suffix immediately before the final extension, requires a nonempty filename -prefix separated by `.`, `-`, `_`, or `~`, and recognizes: - -- hexadecimal suffixes of at least eight characters containing a letter; -- eight-character esbuild-style uppercase Base32 suffixes; -- eight-character Vite/Base64URL-style suffixes with a mixed character class. - -For example, `app.0123abcd.js`, `app-VRTVD5R5.js`, and -`index-DA15JTLU.js` match, while `app.js`, `deadbeef.js`, and -`app.20260714.js` do not. This heuristic is not proof of content addressing; -confirm the publisher's bundler output before enabling a long immutable TTL. A -base rule that matches while this fingerprint check fails emits a debug log with -the rule ID and rejected path. +TTL and either the content-addressed `nextjs-static` preset or an explicit +`fingerprint_style`. + +The filename fingerprint check is intentionally conservative and style-specific. +It examines the suffix immediately before the final extension and requires a +nonempty filename prefix separated by `.`, `-`, `_`, or `~`. Set exactly the +style emitted by the publisher's bundler: + +- `hex`: hexadecimal suffixes of at least eight characters containing a letter, + such as `app.0123abcd.js`; +- `esbuild-base32`: eight-character uppercase Base32 suffixes, such as + `app-VRTVD5R5.js`; +- `vite-base64-url`: eight-character Base64URL suffixes with a mixed character + class, such as `index-BsELY24f.js`. + +A style is an explicit operator assertion, not proof of content addressing. +For example, some human-written mixed-case names can resemble a Vite suffix, so +only select `vite-base64-url` after verifying the publisher's build output. A +base rule that matches while its selected fingerprint style fails emits a debug +log with the rule ID and rejected path. + +Glob patterns are case-sensitive. `*` matches within a single path component, +while `**` matches recursively: `/assets/*.js` matches `/assets/app.js` but not +`/assets/vendor/app.js`; `/assets/**/*.js` matches both. **Next.js preset example** (disabled until the publisher confirms `/_next/static/` is content-addressed): @@ -1112,7 +1119,7 @@ path_globs = [ "/assets/**/*.png", "/assets/**/*.webp", ] -requires_hash_in_filename = true +fingerprint_style = "vite-base64-url" visibility = "public" browser_ttl_seconds = 31536000 edge_ttl_seconds = 31536000 diff --git a/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md b/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md index ce2c37956..8546ebf6e 100644 --- a/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md +++ b/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md @@ -340,7 +340,7 @@ path_globs = [ "/assets/**/*.webp", "/assets/**/*.avif", ] -requires_hash_in_filename = true +fingerprint_style = "vite-base64-url" visibility = "public" browser_ttl_seconds = 31536000 edge_ttl_seconds = 31536000 diff --git a/trusted-server.example.toml b/trusted-server.example.toml index ca566fcb6..ed4afa1cb 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -133,9 +133,9 @@ enabled = false # id = "publisher-fingerprinted-assets" # enabled = false # path_globs = ["/assets/**/*.js", "/assets/**/*.css", "/assets/**/*.png", "/assets/**/*.webp"] -# Immutable custom rules require a supported name-delimited bundler fingerprint -# immediately before the extension, for example app.0123abcd.js or app-VRTVD5R5.js. -# requires_hash_in_filename = true +# Immutable custom rules require an explicit fingerprint_style selected for the +# publisher's bundler, for example "hex", "esbuild-base32", or "vite-base64-url". +# fingerprint_style = "vite-base64-url" # visibility = "public" # browser_ttl_seconds = 31536000 # edge_ttl_seconds = 31536000 From 56083e30c5c90d7df11e589f013f3286eaa1234a Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 14:56:44 -0500 Subject: [PATCH 286/395] Improve publisher HTML cache policy when SSAT is inactive --- crates/trusted-server-core/src/publisher.rs | 85 +++++++++++++++++---- 1 file changed, 72 insertions(+), 13 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4bed98327..02c6a884f 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3003,18 +3003,44 @@ pub async fn handle_publisher_request( // // Gate on `should_run_ad_stack` rather than content-type alone: when no slot // matched, the feature is disabled, or this is not an ad-eligible navigation, - // no per-user `tsjs.adSlots`/`tsjs.bids` are injected, so forcing private - // here would needlessly strip shared cacheability from ordinary publisher - // HTML. Applies regardless of the auction *outcome* (empty bids still inject - // per-user slot state). The separate EC-cookie cache net in the adapter's - // `finalize_response` keeps first-visit identity responses private. + // no per-user `tsjs.adSlots`/`tsjs.bids` are injected. Applies regardless of + // the auction *outcome* (empty bids still inject per-user slot state). The + // separate EC-cookie cache net in the adapter's `finalize_response` keeps + // first-visit identity responses private. let origin_content_type = response .headers() .get(header::CONTENT_TYPE) .and_then(|h| h.to_str().ok()) .unwrap_or_default(); - if should_run_ad_stack && is_html_content_type(origin_content_type) { - enforce_synthesized_html_cache_privacy(&mut response); + if is_html_content_type(origin_content_type) { + if should_run_ad_stack { + enforce_synthesized_html_cache_privacy(&mut response); + } else if is_get + && is_navigation + && !is_prefetch + && !is_bot + && consent_allows_auction + && response.status() == StatusCode::OK + { + // Issue #1007 intentionally caps browser caching for structurally + // inactive server-side ad templates. Request-scoped skips retain the + // origin policy because the same URL can otherwise render templates. + let origin_cache_control = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .map(str::to_ascii_lowercase); + if !origin_cache_control + .as_deref() + .is_some_and(|value| value.contains("private") || value.contains("no-store")) + { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + } + } + } } let content_type = response @@ -4907,13 +4933,16 @@ mod tests { .expect("should build conditional navigation request") } - fn queue_cacheable_html_response(stub: &StubHttpClient) { + fn queue_html_response_with_cache_control( + stub: &StubHttpClient, + cache_control: &'static str, + ) { stub.push_response_with_headers( 200, b"origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), + ("cache-control", cache_control), ("etag", ORIGIN_ETAG), ("last-modified", ORIGIN_LAST_MODIFIED), ("surrogate-control", "max-age=300"), @@ -4983,7 +5012,7 @@ mod tests { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_cacheable_html_response(&stub); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -5076,11 +5105,11 @@ mod tests { } #[tokio::test] - async fn navigation_without_matched_slots_preserves_origin_cache_policy() { + async fn navigation_without_matched_slots_uses_short_browser_cache_policy() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_cacheable_html_response(&stub); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -5126,7 +5155,7 @@ mod tests { ); for (header_name, expected) in [ - (header::CACHE_CONTROL, "public, max-age=300"), + (header::CACHE_CONTROL, "max-age=60"), (header::ETAG, ORIGIN_ETAG), (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), ( @@ -5157,6 +5186,36 @@ mod tests { } } + #[tokio::test] + async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "origin {cache_control} policy should not be weakened" + ); + } + } + #[tokio::test] async fn eligible_navigation_rejects_unexpected_origin_304() { for content_type in [None, Some("text/html; charset=utf-8")] { From d7bfa92aef87cd98acbc45f295091f662b616ae2 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 17:17:40 -0500 Subject: [PATCH 287/395] Document dedicated server-side ad template switch --- ...-server-side-ad-templates-cache-control.md | 238 ++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md diff --git a/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md new file mode 100644 index 000000000..f96bdcb28 --- /dev/null +++ b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md @@ -0,0 +1,238 @@ +# Dedicated Server-Side Ad Templates Switch and Cache Policy Plan + +> **For agentic workers:** Implement this plan task-by-task, keeping the dedicated +> template switch separate from the global auction configuration. + +**Goal:** Add an explicit on/off switch for server-side ad templates, while +retaining the browser-facing cache policy from issue #1007: + +- Server-side ad templates active: `Cache-Control: private, no-store`. +- Server-side ad templates inactive: `Cache-Control: max-age=60`, unless the + origin already sends `private` or `no-store`. +- CDN-specific cache headers must not change when templates are inactive. + +**Issue context:** The current cache-policy change uses the runtime +`should_run_ad_stack` gate. That gate is also affected by `[auction].enabled`, +which is not the right configuration boundary for publisher templates. A +browser can call `POST /auction`, and that endpoint is a separate server-run +auction API. The new switch must disable publisher HTML/page-bids template +delivery without disabling that API. + +## Configuration decision + +Add this field to the existing `[creative_opportunities]` section: + +```toml +[creative_opportunities] +enabled = true +``` + +Use `enabled = false` to turn off server-side ad templates while retaining the +slot definitions and keeping direct `POST /auction` behavior available. + +### Compatibility rules + +- The field defaults to `true` when omitted, preserving existing behavior for + deployments that already have `[creative_opportunities]` configured. +- The section remains optional. An absent section continues to mean that the + feature is unavailable. +- Serialize the default `true` value as omitted, matching the existing + rollback-compatibility pattern for newer creative-opportunity fields. An + explicit `false` must remain serialized so the setting is not silently lost. +- `auction.enabled` remains a separate auction/orchestrator setting. Do not use + it as the dedicated template switch and do not thread the new template flag + into `POST /auction`. + +## Current cache behavior to retain + +The existing HTML policy block in `publisher.rs` must remain structurally +consistent with the current issue #952 behavior: + +1. For an eligible request that runs the server-side ad stack and receives HTML: + - Set `Cache-Control: private, no-store`. + - Remove `ETag` and `Last-Modified`. + - Remove `Surrogate-Control`, `Fastly-Surrogate-Control`, `CDN-Cache-Control`, + and `Cloudflare-CDN-Cache-Control`. +2. For HTML where the server-side ad stack does not run, including an explicit + template disable: + - Read the browser-facing `Cache-Control` header. + - If its value contains `private` or `no-store`, case-insensitively, preserve + the origin value exactly. + - Otherwise set exactly `Cache-Control: max-age=60`. + - Leave validators and all CDN-specific cache headers untouched. +3. Preserve the later adapter response-privacy finalization for cookie-bearing + responses; this plan does not refactor that behavior. + +## File map + +### Configuration and compatibility + +- `crates/trusted-server-core/src/creative_opportunities.rs` + - Add `CreativeOpportunitiesConfig::enabled` with a default-true serde + implementation and documentation. + - Add a small accessor if it improves readability, but keep the source of + truth in this config type. + - Update config constructors and serialization tests. +- `crates/trusted-server-core/src/settings.rs` + - Keep `creative_opportunities` parsing and runtime preparation compatible with + the new field. + - Make `creative_opportunity_slots()` return an empty slice when the section + is absent or explicitly disabled, so all adapters receive one consistent + runtime view. + - Add TOML and environment-override coverage for `enabled = false`. +- `crates/trusted-server-core/src/config.rs` + - Extend legacy-schema tests to prove default `enabled = true` is omitted from + serialized blobs and remains readable by older binaries. + - Prove an explicit `enabled = false` is serialized, making rollback failure + loud rather than silently re-enabling templates. +- `trusted-server.example.toml` + - Document `creative_opportunities.enabled` and show how to turn templates off + without deleting slot definitions. +- `docs/guide/configuration.md` + - Add the field to the creative-opportunities reference and document the + environment override: + `TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false`. + - Clarify that this switch controls publisher HTML/page-bids template + delivery, not direct `POST /auction` callers. +- `CHANGELOG.md` + - Add an entry describing the dedicated template switch and cache behavior. + +### Publisher execution and cache policy + +- `crates/trusted-server-core/src/publisher.rs` + - Include the dedicated flag in the initial publisher eligibility decision. + - Do not match, dispatch, or inject server-side ad templates when the flag is + false, even if slots are configured and `[auction].enabled` is true. + - Apply the issue #1007 inactive-HTML cache policy in this state. + - Update skip-reason diagnostics/telemetry so `ad_templates_disabled` is + distinguishable from `auction_disabled`, consent denial, bots, prefetch, and + no matching slots. + - Update `handle_page_bids` so an explicit template disable returns the normal + empty JSON shape (`slots: []`, `bids: {}`) rather than slot definitions. Keep + the current `404` behavior for an absent `[creative_opportunities]` section. + - Extend the existing SSAT cache-policy and eligibility tests. +- `crates/trusted-server-core/src/auction/endpoints.rs` + - Do not gate `POST /auction` on the new template flag. + - Add a regression test or test fixture proving that disabling + `creative_opportunities.enabled` does not suppress a direct auction request + when providers are configured. + - Separately document/verify the existing behavior of `[auction].enabled` for + this endpoint; do not conflate that global setting with the new template + switch. + +### Adapter propagation and browser behavior + +The adapters already pass `Settings::creative_opportunity_slots()` into the +publisher/page-bids handlers. Update and verify these call sites so the central +empty-slice behavior is honored; avoid adding four divergent config checks: + +- `crates/trusted-server-adapter-fastly/src/app.rs` +- `crates/trusted-server-adapter-axum/src/app.rs` +- `crates/trusted-server-adapter-cloudflare/src/app.rs` +- `crates/trusted-server-adapter-spin/src/app.rs` + +No route-level flag is needed if the core `Settings` accessor and handlers are +correct. Add adapter route assertions only where existing fixtures make them +useful. + +The browser runtime already defaults `window.tsjs.adSlots` and +`window.tsjs.bids` to empty values when the edge does not inject templates. If +terminology is updated, adjust these comments/tests without changing runtime +semantics: + +- `crates/trusted-server-js/lib/src/core/index.ts` +- `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Relevant page-bids tests under `crates/trusted-server-js/lib/test/integrations/gpt/` + +## Implementation tasks + +### Task 1: Add and serialize the dedicated setting + +- [ ] Add `enabled: bool` to `CreativeOpportunitiesConfig` with default `true`. +- [ ] Use `skip_serializing_if` so the default value does not appear in stored + config blobs; explicit `false` must serialize. +- [ ] Update all Rust struct literals in `creative_opportunities.rs` and + `publisher.rs` tests. +- [ ] Add parsing, default, false-value, and environment-override tests. +- [ ] Update the legacy compatibility tests in `config.rs`. + +### Task 2: Thread the setting through publisher eligibility + +- [ ] Update `should_run_server_side_ad_stack` to accept the dedicated template + flag as an explicit gate, with a descriptive parameter/doc comment. +- [ ] Ensure initial publisher slot matching and `Settings::creative_opportunity_slots` + do not expose slots when templates are disabled. +- [ ] Preserve the existing `[auction].enabled` and consent gates as separate + conditions. +- [ ] Add an `ad_templates_disabled` diagnostic/telemetry skip reason where the + current branch records a skipped auction. + +### Task 3: Apply the cache policy to the dedicated-off state + +- [ ] Keep the active-SSAT `private, no-store` behavior and validator/CDN header + removal unchanged. +- [ ] Keep the inactive-HTML `max-age=60` behavior from issue #1007. +- [ ] Verify that explicit template disable changes only browser-facing + `Cache-Control` for cacheable HTML; preserve `ETag`, `Last-Modified`, and + every CDN-specific header. +- [ ] Verify that origin `private`, `PRIVATE`, `no-store`, and `No-Store` values + remain unchanged. + +### Task 4: Gate SPA page-bids/template delivery + +- [ ] Include `co_config.enabled` in the `ad_stack_enabled` decision in + `handle_page_bids`. +- [ ] Return empty slots and bids for an explicit disable while retaining the + endpoint and its existing response privacy headers. +- [ ] Keep the absent-section `404` behavior unchanged. +- [ ] Add tests for enabled, disabled, absent, consent-denied, bot, and prefetch + cases as appropriate; preserve existing tests for `[auction].enabled=false`. + +### Task 5: Protect direct `POST /auction` from accidental coupling + +- [ ] Add a focused endpoint test with `creative_opportunities.enabled=false` + and a recording provider. +- [ ] Assert that the provider still sees the direct auction request and that + the response remains a normal OpenRTB response. +- [ ] If the test reveals that `[auction].enabled=false` also needs a separate + product decision for `/auction`, record that as a follow-up rather than + changing it as part of the template-switch work. + +### Task 6: Update docs, examples, comments, and adapter coverage + +- [ ] Update the example config, configuration guide, and changelog. +- [ ] Update stale comments that call `[auction].enabled` the universal template + kill switch. +- [ ] Verify all four adapter call sites use the centralized disabled-slot view. +- [ ] Run JS tests if comments or tests are touched; no JS behavior change is + expected. + +## Test plan + +Use target-matched commands; do not run bare workspace tests because the +workspace contains multiple runtime targets. + +- [ ] `cargo test-axum -p trusted-server-core publisher` +- [ ] `cargo test-fastly` +- [ ] `cargo test-axum` +- [ ] `cargo test-cloudflare` +- [ ] `cargo test-spin` +- [ ] `cargo fmt --all -- --check` +- [ ] `cargo clippy-fastly` +- [ ] `cargo clippy-axum` +- [ ] `cargo clippy-cloudflare` +- [ ] `cargo clippy-cloudflare-wasm` +- [ ] `cargo clippy-spin-native` +- [ ] `cargo clippy-spin-wasm` +- [ ] `cd crates/trusted-server-js/lib && npx vitest run` if JS tests/comments change +- [ ] `cd docs && npm run format` if documentation formatting is required + +## Non-goals + +- Do not change CDN-specific cache policy for inactive templates. +- Do not change adapter response privacy or cookie handling. +- Do not use `auction.rewrite_creatives` as the template switch; it controls + creative URL rewriting, not whether the server-side template stack runs. +- Do not gate or disable direct `POST /auction` as part of this feature. +- Do not remove slot definitions when the switch is off; the point of the switch + is to provide a reversible runtime control. From e9a55d4a3cdc8f3b97e675c44fd1649651ae176c Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 18:03:25 -0500 Subject: [PATCH 288/395] Add dedicated server-side ad template switch --- CHANGELOG.md | 1 + .../src/auction/endpoints.rs | 127 +++++++++- crates/trusted-server-core/src/config.rs | 27 ++ .../src/creative_opportunities.rs | 47 +++- crates/trusted-server-core/src/publisher.rs | 237 +++++++++++++++--- crates/trusted-server-core/src/settings.rs | 46 +++- .../trusted-server-js/lib/src/core/index.ts | 8 +- .../lib/src/integrations/gpt/index.ts | 12 +- .../test/integrations/gpt/spa_hook.test.ts | 6 +- docs/guide/configuration.md | 20 +- trusted-server.example.toml | 3 + 11 files changed, 480 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c00487769..a221f7af5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Publisher HTML uses `Cache-Control: max-age=60` for successful GET document responses when server-side ad templates are inactive, intentionally replacing the origin browser cache policy while preserving CDN-specific cache headers. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers. An explicit `enabled = false` is not compatible with older binaries: restore the default, re-push and finalize the config before rolling back. - **Breaking** — Replaced the legacy APS contextual integration with APS OpenRTB at `/e/pb/bid`. APS configuration now uses canonical `account_id` (`pub_id` remains a compatibility alias), no longer requires APS-specific slot IDs, and defaults script creative eligibility off. Operators must update the endpoint, disable native APS demand for Trusted Server cohorts, and prepare GAM/Universal Creative targeting for `hb_bidder=aps` before rollout. `aps` entries in Prebid bidder lists are logged and stripped. APS renderer winners now preserve the upstream bid `id`, omit `crid` when APS omits it, and carry `ext.trusted_server.renderer` instead of `adm`; external `/auction` consumers must support this response shape. - **Breaking** — All auction paths now forward only a validated publisher-owned page URL as `site.page`, removing query and fragment data. APS OpenRTB omits `site.ref`; the existing Prebid Server path continues to forward the browser `Referer` as `site.ref`. Query-driven sites may lose contextual targeting and per-page reporting signals that previously came from query parameters. - **Breaking** — `bid_param_zone_overrides` inner values must now be JSON objects; previously non-object or empty values (`"header" = "x"`, `"header" = {}`) were accepted and silently produced a dead rule at runtime. They now fail at startup with a configuration error. Operators upgrading should audit their `bid_param_zone_overrides` config for non-object zone entries. diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index c4af6fd3d..bcae50896 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -587,10 +587,14 @@ mod tests { use crate::consent::types::ConsentContext; use crate::openrtb::Uid; use crate::platform::test_support::{ - NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, noop_services, + NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, StubHttpClient, + noop_services, }; - use crate::platform::{ClientInfo, PlatformResponse}; - use crate::test_support::tests::create_test_settings; + use crate::platform::{ + ClientInfo, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, + PlatformResponse, + }; + use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use serde_json::json; @@ -675,6 +679,123 @@ mod tests { } } + /// Provider used to prove that direct `/auction` remains available when + /// publisher server-side ad templates are disabled. + struct TemplateSwitchProbeProvider { + calls: Arc>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for TemplateSwitchProbeProvider { + fn provider_name(&self) -> &'static str { + "template_switch_probe" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + *self.calls.lock().expect("should lock provider call count") += 1; + let request = Request::builder() + .method("POST") + .uri("https://bidder.example/auction") + .body(EdgeBody::empty()) + .expect("should build probe provider request"); + context + .services + .http_client() + .send_async(PlatformHttpRequest::new( + request, + "template-switch-probe-backend", + )) + .await + .change_context(TrustedServerError::Auction { + message: "probe provider launch failed".to_string(), + }) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::success( + self.provider_name(), + Vec::new(), + 0, + )) + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("template-switch-probe-backend".to_string()) + } + } + + #[tokio::test] + async fn direct_auction_remains_available_when_templates_are_disabled() { + let settings_toml = format!( + "{}\n[auction]\nenabled = true\nproviders = [\"template_switch_probe\"]\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&settings_toml) + .expect("should parse settings with disabled templates"); + let calls = Arc::new(Mutex::new(0)); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(TemplateSwitchProbeProvider { + calls: Arc::clone(&calls), + })); + + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"probe response".to_vec()); + let services = RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::clone(&stub) as Arc) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .build(); + let ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let body = json!({ + "adUnits": [{ + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + }); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize body"), + )) + .expect("should build auction request"); + + let response = handle_auction( + &settings, + &orchestrator, + None, + None, + &ec_context, + &services, + req, + ) + .await + .expect("direct auction should remain available"); + + assert_eq!( + *calls.lock().expect("should lock provider call count"), + 1, + "disabling publisher templates must not disable direct /auction" + ); + assert_eq!(response.status(), StatusCode::OK); + } + #[tokio::test] async fn auction_endpoint_consent_gate_returns_no_bid_without_contacting_providers() { // GDPR/unknown jurisdiction lacking effective TCF Purpose 1 must not run diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index e74ef4150..da82e7581 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -323,10 +323,37 @@ formats = [{ width = 300, height = 250 }] fn absent_gam_unit_template_is_accepted_by_legacy_schema() { let creative_opportunities = serialized_creative_opportunities(None); + assert!( + creative_opportunities.get("enabled").is_none(), + "default template switch should be omitted for legacy binaries" + ); serde_json::from_value::(creative_opportunities) .expect("should accept absent GAM unit template"); } + #[test] + fn disabled_creative_opportunities_flag_is_visible_to_legacy_schema() { + let mut toml = crate_test_settings_str(); + toml.push_str( + r#" + +[creative_opportunities] +enabled = false +gam_network_id = "99999" +"#, + ); + let app_config: TrustedServerAppConfig = + toml::from_str(&toml).expect("should deserialize app config wrapper"); + let creative_opportunities = serde_json::to_value(app_config) + .expect("should serialize app config wrapper") + .get("creative_opportunities") + .cloned() + .expect("should contain creative opportunities"); + + serde_json::from_value::(creative_opportunities) + .expect_err("legacy binaries should reject an explicit disabled switch"); + } + #[test] fn deploy_validation_rejects_placeholders() { let settings = Settings::from_toml( diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index e44b0cbcf..cd11e1c14 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -183,10 +183,27 @@ fn derive_section(path: &str, section_root: &str, section_segment: usize) -> Str } } +const fn default_enabled() -> bool { + true +} + +const fn is_default_enabled(value: &bool) -> bool { + *value == default_enabled() +} + /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct CreativeOpportunitiesConfig { + /// Enables server-side ad template delivery on publisher HTML and page-bids requests. + /// + /// This does not disable the direct `POST /auction` endpoint. The default is + /// `true` so existing creative-opportunity configurations retain their behavior. + #[serde( + default = "default_enabled", + skip_serializing_if = "is_default_enabled" + )] + pub enabled: bool, /// GAM network ID used to build default unit paths. pub gam_network_id: String, /// Maximum time in milliseconds to wait for the server-side auction before @@ -244,7 +261,7 @@ pub struct CreativeOpportunitiesConfig { /// [`section_root`](Self::section_root) are omitted. #[serde(default, skip_serializing_if = "Option::is_none")] pub section_segment: Option, - /// Slot templates. Empty vec = feature disabled (no auction fired, no globals injected). + /// Slot templates. An empty vec or `enabled = false` disables template delivery. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub slot: Vec, } @@ -1143,12 +1160,39 @@ mod tests { assert_eq!(derive_section("/%%%/x", "home", 0), "_"); } + #[test] + fn enabled_defaults_true_and_is_omitted_from_serialized_config() { + let config = make_config_with_section_template(None); + assert!( + config.enabled, + "template delivery should default to enabled" + ); + let value = serde_json::to_value(&config).expect("should serialize config"); + assert!( + value.get("enabled").is_none(), + "default enabled value should be omitted for rollback compatibility" + ); + } + + #[test] + fn disabled_template_switch_is_serialized() { + let mut config = make_config_with_section_template(None); + config.enabled = false; + let value = serde_json::to_value(&config).expect("should serialize config"); + assert_eq!( + value.get("enabled"), + Some(&serde_json::Value::Bool(false)), + "explicitly disabled template delivery must remain in config blobs" + ); + } + fn make_config_with_section_template( section_root: Option<&str>, ) -> CreativeOpportunitiesConfig { let mut slot = make_slot("ad-header-0", vec!["/news/*"]); slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "99999".to_string(), auction_timeout_ms: None, price_granularity: PriceGranularity::default(), @@ -1546,6 +1590,7 @@ mod tests { // Older binaries deserialize this struct with `deny_unknown_fields`, so // a pushed config blob must not carry `"section_root": null`. let config = CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "99999".to_string(), auction_timeout_ms: None, price_granularity: PriceGranularity::default(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 02c6a884f..1ac20731d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1800,27 +1800,34 @@ pub(crate) fn is_prefetch_request(req: &Request) -> bool { header("sec-purpose") || header("purpose") } -/// Returns true only when the publisher request should run the full -/// server-side ad stack: auction dispatch plus initial ad-slot injection. +#[derive(Debug, Clone, Copy)] +struct ServerSideAdStackConfig { + /// Dedicated `[creative_opportunities].enabled` switch. + ad_templates_enabled: bool, + /// Global `[auction].enabled` gate used by publisher/page-bids flows. + auction_enabled: bool, +} + +/// Returns true only when the publisher should inject and run server-side ad templates. /// -/// `auction_enabled` is the global `[auction].enabled` kill switch — when -/// false, no automatic server-side auction or ad-slot injection runs. -pub(crate) fn should_run_server_side_ad_stack( +/// This includes auction dispatch plus initial ad-slot injection. +fn should_run_server_side_ad_stack( is_get: bool, is_navigation: bool, is_prefetch: bool, is_bot: bool, has_matched_slots: bool, consent_allows_auction: bool, - auction_enabled: bool, + config: ServerSideAdStackConfig, ) -> bool { is_get && is_navigation && !is_prefetch && !is_bot + && config.ad_templates_enabled && has_matched_slots && consent_allows_auction - && auction_enabled + && config.auction_enabled } /// Write winning bids from an auction result into the shared `ad_bids_state` lock. @@ -2697,7 +2704,10 @@ pub async fn handle_publisher_request( let is_prefetch = is_prefetch_request(&req); let is_bot = is_bot_user_agent(&req); - let matched_slots = if is_get { + let creative_opportunities = settings.creative_opportunities.as_ref(); + let ad_templates_enabled = creative_opportunities.is_some_and(|co_config| co_config.enabled); + let ad_templates_disabled = creative_opportunities.is_some_and(|co_config| !co_config.enabled); + let matched_slots = if is_get && ad_templates_enabled { settings .creative_opportunities .as_ref() @@ -2720,7 +2730,10 @@ pub async fn handle_publisher_request( is_bot, !matched_slots.is_empty(), consent_allows_auction, - auction.orchestrator.is_enabled(), + ServerSideAdStackConfig { + ad_templates_enabled, + auction_enabled: auction.orchestrator.is_enabled(), + }, ); let should_run_auction = should_run_ad_stack; // Diagnostic: shows which gate suppresses the server-side auction. Pair with @@ -2728,14 +2741,16 @@ pub async fn handle_publisher_request( // when `consent_allows_auction=false`. log::debug!( "server-side ad-stack gate: is_get={is_get} is_navigation={is_navigation} \ - is_prefetch={is_prefetch} is_bot={is_bot} matched_slots={} \ - consent_allows_auction={consent_allows_auction} orchestrator_enabled={} \ - -> should_run_auction={should_run_auction}", + is_prefetch={is_prefetch} is_bot={is_bot} ad_templates_enabled={ad_templates_enabled} \ + matched_slots={} consent_allows_auction={consent_allows_auction} \ + orchestrator_enabled={} -> should_run_auction={should_run_auction}", matched_slots.len(), auction.orchestrator.is_enabled(), ); - if matched_slots.is_empty() && settings.creative_opportunities.is_some() { + if ad_templates_disabled { + log::debug!("Server-side ad templates are disabled by configuration"); + } else if matched_slots.is_empty() && settings.creative_opportunities.is_some() { log::debug!( "No creative opportunity slots matched path '{}' — skipping auction and injection", request_path @@ -2860,7 +2875,9 @@ pub async fn handle_publisher_request( } } } else { - let skip_reason = if !auction.orchestrator.is_enabled() { + let skip_reason = if ad_templates_disabled { + "ad_templates_disabled" + } else if !auction.orchestrator.is_enabled() { "auction_disabled" } else if !consent_allows_auction { "consent_denied" @@ -3971,7 +3988,11 @@ pub async fn handle_page_bids( }) .unwrap_or_else(|| "/".to_string()); - let matched_slots = match_renderable_slots(auction.slots, co_config, &path_param); + let matched_slots = if co_config.enabled { + match_renderable_slots(auction.slots, co_config, &path_param) + } else { + Vec::new() + }; let request_info = crate::http_util::RequestInfo::from_request(&req, services.client_info()); let ec_id = ec_context.ec_value().filter(|_| ec_context.ec_allowed()); @@ -3990,7 +4011,10 @@ pub async fn handle_page_bids( let is_bot = is_bot_user_agent(&req); let auction_enabled = auction.orchestrator.is_enabled(); - if !auction_enabled { + let ad_templates_enabled = co_config.enabled; + if !ad_templates_enabled { + log::debug!("page-bids: [creative_opportunities].enabled is false — skipping templates"); + } else if !auction_enabled { log::debug!("page-bids: [auction].enabled is false — skipping auction"); } else if matched_slots.is_empty() { log::debug!( @@ -4006,14 +4030,14 @@ pub async fn handle_page_bids( ); } - // The [auction].enabled kill switch and a consent denial disable the entire - // server-side ad stack. In those states the endpoint must return no slots, - // so the SPA hook does not assign `ts.adSlots` and call `adInit()` — - // otherwise the kill switch/consent gate would stop SSP calls but still let - // the client create/refresh GPT slots. Bot/prefetch requests, by contrast, - // keep their slot definitions (the placement structure is unchanged) but - // skip the live auction, matching the existing bot/prefetch behaviour. - let ad_stack_enabled = auction_enabled && consent_allows_auction; + // The dedicated template switch, [auction].enabled, and a consent denial + // disable the entire server-side ad stack. In those states the endpoint must + // return no slots, so the SPA hook does not assign `ts.adSlots` and call + // `adInit()` — otherwise the gate would stop SSP calls but still let the + // client create/refresh GPT slots client-side. Bot/prefetch requests, by + // contrast, keep their slot definitions (the placement structure is + // unchanged) but skip the live auction, matching the existing behavior. + let ad_stack_enabled = ad_templates_enabled && auction_enabled && consent_allows_auction; let (winning_bids, prebuilt_bid_map) = if matched_slots.is_empty() { (std::collections::HashMap::new(), None) @@ -4114,7 +4138,9 @@ pub async fn handle_page_bids( } } } else { - let skip_reason = if !auction_enabled { + let skip_reason = if !ad_templates_enabled { + "ad_templates_disabled" + } else if !auction_enabled { "auction_disabled" } else if !consent_allows_auction { "consent_denied" @@ -4875,6 +4901,15 @@ mod tests { .expect("should parse settings with auction and creative opportunities enabled") } + fn settings_with_disabled_ad_templates() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n\ + [creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled ad templates") + } + fn settings_with_dispatching_provider() -> Settings { let toml = format!( "{}\n[auction]\nenabled = true\nproviders = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n\ @@ -5186,6 +5221,72 @@ mod tests { } } + #[tokio::test] + async fn disabled_ad_templates_use_short_browser_cache_policy() { + // Arrange + let settings = settings_with_disabled_ad_templates(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let response = run_with_slots( + &settings, + &services, + &slots, + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "disabled server-side ad templates should not bypass the origin cache" + ); + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("max-age=60"), + "disabled server-side ad templates should use the short browser cache policy" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "disabled server-side ad templates should preserve {header_name}" + ); + } + } + #[tokio::test] async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { let settings = settings_with_enabled_auction_and_creative_opportunities(); @@ -6043,39 +6144,69 @@ mod tests { #[test] fn server_side_ad_stack_runs_only_when_all_auction_gates_pass() { + let enabled_config = ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: true, + }; assert!( - should_run_server_side_ad_stack(true, true, false, false, true, true, true), - "GET, real navigation, matched slots, and consent should run TS ad stack" + should_run_server_side_ad_stack(true, true, false, false, true, true, enabled_config,), + "GET, real navigation, enabled templates, matched slots, and consent should run TS ad stack" ); assert!( - !should_run_server_side_ad_stack(false, true, false, false, true, true, true), + !should_run_server_side_ad_stack(false, true, false, false, true, true, enabled_config,), "non-GET requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, false, false, false, true, true, true), + !should_run_server_side_ad_stack(true, false, false, false, true, true, enabled_config,), "non-document requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, true, false, true, true, true), + !should_run_server_side_ad_stack(true, true, true, false, true, true, enabled_config,), "prefetch requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, true, true, true, true), + !should_run_server_side_ad_stack(true, true, false, true, true, true, enabled_config,), "bot requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, false, true, true), + !should_run_server_side_ad_stack(true, true, false, false, false, true, enabled_config,), "requests with no matching slots should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, false, true), + !should_run_server_side_ad_stack(true, true, false, false, true, false, enabled_config,), "requests without required consent should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, true, false), + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: false, + }, + ), "disabled [auction].enabled kill switch should skip TS ad stack and injection" ); + assert!( + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: false, + auction_enabled: true, + }, + ), + "disabled [creative_opportunities].enabled switch should skip TS ad stack and injection" + ); } #[tokio::test] @@ -8733,6 +8864,7 @@ mod tests { fn make_config() -> CreativeOpportunitiesConfig { CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "21765378893".to_string(), auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, @@ -10412,6 +10544,14 @@ mod tests { Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } + fn settings_with_co_templates_disabled() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled templates") + } + async fn run_page_bids( settings: &Settings, orchestrator: &AuctionOrchestrator, @@ -11161,6 +11301,35 @@ mod tests { ); } + #[tokio::test] + async fn disabled_server_side_ad_templates_return_no_slots_or_bids() { + // The dedicated template switch must suppress publisher/page-bids + // delivery without using the global auction switch. + let settings = settings_with_co_templates_disabled(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let req = make_page_bids_request("/2024/01/my-article/"); + + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 0, + "disabled server-side ad templates must not return slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "disabled server-side ad templates must not produce bids" + ); + } + #[tokio::test] async fn consent_denied_returns_no_slots_or_bids() { // When consent denies the server-side auction (here: Jurisdiction diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 598c00056..81a2323ea 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2095,13 +2095,14 @@ impl Settings { Ok(()) } - /// Returns compiled creative opportunity slots, or empty slice if feature is disabled. + /// Returns compiled creative opportunity slots when template delivery is enabled. #[must_use] pub fn creative_opportunity_slots( &self, ) -> &[crate::creative_opportunities::CreativeOpportunitySlot] { self.creative_opportunities .as_ref() + .filter(|co| co.enabled) .map(|co| co.slot.as_slice()) .unwrap_or(&[]) } @@ -5014,6 +5015,10 @@ formats = [{ width = 300, height = 250 }] let co = settings .creative_opportunities .expect("should have creative_opportunities"); + assert!( + co.enabled, + "creative-opportunity templates should default to enabled" + ); assert_eq!(co.gam_network_id, "21765378893"); assert_eq!(co.auction_timeout_ms, Some(500)); assert_eq!( @@ -5023,6 +5028,45 @@ formats = [{ width = 300, height = 250 }] ); } + #[test] + fn settings_disables_creative_opportunity_slots_when_configured_off() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = false\ngam_network_id = \"21765378893\"\n\n[[creative_opportunities.slot]]\nid = \"atf\"\npage_patterns = [\"/\"]\nformats = [{{ width = 300, height = 250 }}]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("should parse disabled templates"); + assert!( + settings.creative_opportunity_slots().is_empty(), + "disabled template delivery should expose no runtime slots" + ); + } + + #[test] + fn settings_creative_opportunity_enabled_flag_supports_environment_override() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = true\ngam_network_id = \"21765378893\"\n", + crate_test_settings_str() + ); + let env_key = format!( + "{}{}CREATIVE_OPPORTUNITIES{}ENABLED", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + + temp_env::with_var(env_key, Some("false"), || { + let settings = Settings::from_toml_and_env(&toml) + .expect("should parse template enabled environment override"); + assert!( + !settings + .creative_opportunities + .expect("should have creative opportunities") + .enabled, + "environment override should disable template delivery" + ); + }); + } + #[test] fn settings_rejects_invalid_creative_opportunity_slot_id() { let toml = r#" diff --git a/crates/trusted-server-js/lib/src/core/index.ts b/crates/trusted-server-js/lib/src/core/index.ts index 5d8e41971..b2c4e41e1 100644 --- a/crates/trusted-server-js/lib/src/core/index.ts +++ b/crates/trusted-server-js/lib/src/core/index.ts @@ -36,10 +36,10 @@ api.getConfig = getConfig; // Provide core requestAds API api.requestAds = requestAds; // Defensive defaults: the edge injects adSlots (head-open) and bids (before -// ) only when the server-side ad stack runs for the request. When it -// is gated off (kill switch, consent fail-closed, bots, prefetch), page code -// reading window.tsjs.bids / window.tsjs.adSlots must still see defined -// values instead of throwing. Injected scripts overwrite these wholesale. +// ) only when server-side ad templates run for the request. When template +// delivery is disabled or gated off (auction/consent, bots, prefetch), page code +// reading window.tsjs.bids / window.tsjs.adSlots must still see defined values +// instead of throwing. Injected scripts overwrite these wholesale. api.adSlots ??= []; api.bids ??= {}; // Point global tsjs diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index f0df35974..ebf7e225e 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1094,8 +1094,8 @@ export function installTsAdInit(): void { ts.prevSlotTargetingKeys = nextSlotTargetingKeys; // Whether this call produced any TS slot to render. A gated page-bids - // response (auction kill switch or consent denial) returns no slots, so - // the loops above leave these empty. + // response (template switch, auction gate, or consent denial) returns no + // slots, so the loops above leave these empty. const hasRenderableWork = slotsToDisplay.length > 0 || slotsToRefresh.length > 0; // enableSingleRequest and enableServices must only be called once per page @@ -1403,10 +1403,10 @@ export function installSpaAuctionHook(): void { // This route is now the committed, loaded state — a later failed // navigation rolls back here, and a return trip no-ops correctly. lastAppliedPath = path; - // An empty page-bids response (auction kill switch or consent gate) carries - // no TS slots. Only run adInit() when there are slots to apply or prior TS - // state to sweep — otherwise a consent-denied or kill-switched navigation - // must not enter the GPT command queue and risk activating services. + // An empty page-bids response (template switch, auction, or consent gate) + // carries no TS slots. Only run adInit() when there are slots to apply or + // prior TS state to sweep — otherwise a gated navigation must not enter + // the GPT command queue and risk activating services. const hasPriorTsState = (ts.prevGptSlots?.length ?? 0) > 0 || Object.keys(ts.prevSlotTargetingKeys ?? {}).length > 0 || diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 7127efcb4..314348fa8 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -216,9 +216,9 @@ describe('installSpaAuctionHook', () => { }); it('skips adInit on an empty page-bids response with no prior TS state', async () => { - // A gated page-bids response (auction kill switch or consent denial) returns - // no slots. With no prior TS state to sweep, the hook must not call adInit() - // so a consent-denied navigation cannot activate the publisher's GPT setup. + // A gated page-bids response (template switch, auction gate, or consent + // denial) returns no slots. With no prior TS state to sweep, the hook must + // not call adInit() so a gated navigation cannot activate publisher GPT. fetchStub.mockResolvedValue({ ok: true, json: async () => ({ slots: [], bids: {} }), diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index ddb6544ce..b763cac24 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1347,8 +1347,16 @@ Defines the ad slots the trusted server offers on a page: which pages each slot appears on (`page_patterns`), its supported sizes (`formats`), and the GAM ad unit it maps to (`gam_unit_path`). +`enabled` is the dedicated server-side ad-template switch. It defaults to `true` +for compatibility with existing configurations. Set it to `false` to stop +publisher HTML and SPA page-bids template delivery while retaining the slot +configuration and direct `POST /auction` endpoint. The browser-facing cache +policy for a disabled template stack is `Cache-Control: max-age=60`, unless the +origin already sends `private` or `no-store`. + ```toml [creative_opportunities] +enabled = true # set to false to disable server-side ad templates gam_network_id = "123456789" price_granularity = "dense" @@ -1367,6 +1375,13 @@ page_patterns = ["/", "/news", "/news/*", "/reviews", "/reviews/*"] formats = [{ width = 728, height = 90 }] ``` +The same switch can be overridden through the legacy environment-variable +loader: + +```bash +TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false +``` + ### `gam_unit_path` templating `gam_unit_path` is a template. A publisher whose ad unit varies by site section @@ -1420,8 +1435,9 @@ publisher-specific. Startup fails if `{section}` is used without a valid `section_root`. Startup rejects a blank `gam_network_id` only when an absent path/default or a `{network_id}` template consumes it; static paths and templates without `{network_id}` do not consume it. A -`[creative_opportunities]` block with no slots is disabled, so its -`gam_network_id` is not checked. +`[creative_opportunities]` block with `enabled = false` or no slots is +inactive, so no publisher templates are delivered and its `gam_network_id` is +not checked when no slot uses it. Both knobs are config-driven, so the URL→section convention stays with the publisher: `section_segment` selects which segment names the section, and diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 19ecda4a5..ed20c4083 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -181,6 +181,9 @@ ja4_endpoint_enabled = false auction_html_comment = false [creative_opportunities] +# Set to false to disable server-side ad templates while retaining slot definitions +# and direct POST /auction callers. +enabled = true gam_network_id = "123456789" # FCP is not affected by this value — body content above has already # streamed and painted before the hold begins. What this caps is the slip on From 03e429a58428c6610fbf6c3e60847e5be0e8213d Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 13 Aug 2026 15:03:54 -0500 Subject: [PATCH 289/395] Address cache policy review feedback --- .../src/auction/endpoints.rs | 8 +- crates/trusted-server-core/src/config.rs | 2 +- crates/trusted-server-core/src/publisher.rs | 175 +++++++++++++++++- docs/guide/configuration.md | 16 +- ...-server-side-ad-templates-cache-control.md | 35 ++-- trusted-server.example.toml | 4 +- 6 files changed, 207 insertions(+), 33 deletions(-) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index bcae50896..771dacb51 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -590,10 +590,7 @@ mod tests { NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, StubHttpClient, noop_services, }; - use crate::platform::{ - ClientInfo, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, - PlatformResponse, - }; + use crate::platform::{ClientInfo, PlatformHttpClient, PlatformHttpRequest, PlatformResponse}; use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; @@ -695,7 +692,7 @@ mod tests { &self, _request: &AuctionRequest, context: &AuctionContext<'_>, - ) -> Result> { + ) -> Result> { *self.calls.lock().expect("should lock provider call count") += 1; let request = Request::builder() .method("POST") @@ -713,6 +710,7 @@ mod tests { .change_context(TrustedServerError::Auction { message: "probe provider launch failed".to_string(), }) + .map(ProviderRequestOutcome::pending) } async fn parse_response( diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index da82e7581..818b6fcc5 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -332,7 +332,7 @@ formats = [{ width = 300, height = 250 }] } #[test] - fn disabled_creative_opportunities_flag_is_visible_to_legacy_schema() { + fn disabled_creative_opportunities_flag_is_rejected_by_legacy_schema() { let mut toml = crate_test_settings_str(); toml.push_str( r#" diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1ac20731d..055cb81de 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3003,8 +3003,6 @@ pub async fn handle_publisher_request( return Ok(PublisherResponse::Buffered(response)); } - crate::integrations::gpt_diagnostics::finalize_response(&gpt_diagnostics, &mut response); - let ad_slots_script = if should_run_ad_stack { settings .creative_opportunities @@ -3058,7 +3056,8 @@ pub async fn handle_publisher_request( } } } - } + + crate::integrations::gpt_diagnostics::finalize_response(&gpt_diagnostics, &mut response); let content_type = response .headers() @@ -4910,6 +4909,11 @@ mod tests { Settings::from_toml(&toml).expect("should parse settings with disabled ad templates") } + fn settings_without_creative_opportunities() -> Settings { + Settings::from_toml(&crate_test_settings_str()) + .expect("should parse settings without creative opportunities") + } + fn settings_with_dispatching_provider() -> Settings { let toml = format!( "{}\n[auction]\nenabled = true\nproviders = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n\ @@ -4971,9 +4975,17 @@ mod tests { fn queue_html_response_with_cache_control( stub: &StubHttpClient, cache_control: &'static str, + ) { + queue_html_response_with_status_and_cache_control(stub, 200, cache_control); + } + + fn queue_html_response_with_status_and_cache_control( + stub: &StubHttpClient, + status: u16, + cache_control: &'static str, ) { stub.push_response_with_headers( - 200, + status, b"origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), @@ -5288,10 +5300,17 @@ mod tests { } #[tokio::test] - async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { + async fn navigation_without_matched_slots_replaces_origin_cache_policy() { let settings = settings_with_enabled_auction_and_creative_opportunities(); - for cache_control in ["private, max-age=0", "No-Store"] { + for cache_control in [ + "no-cache", + "max-age=0", + "must-revalidate", + "s-maxage=0", + "private, max-age=0", + "No-Store", + ] { // Arrange let stub = Arc::new(StubHttpClient::new()); queue_html_response_with_cache_control(&stub, cache_control); @@ -5311,8 +5330,148 @@ mod tests { .headers .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some(cache_control), - "origin {cache_control} policy should not be weakened" + Some("max-age=60"), + "inactive server-side ad templates should replace origin {cache_control} policy" + ); + } + } + + #[tokio::test] + async fn absent_creative_opportunities_use_short_browser_cache_policy() { + // Arrange + let settings = settings_without_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("max-age=60"), + "absent creative opportunities should be treated as an inactive server-side ad stack" + ); + } + + #[tokio::test] + async fn inactive_ad_stack_preserves_non_ok_response_cache_policy() { + let settings = settings_with_disabled_ad_templates(); + + for status in [206, 404, 500, 503] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_status_and_cache_control(&stub, status, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = run_with_slots( + &settings, + &services, + &[article_slot()], + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-cache"), + "inactive server-side ad templates should preserve origin policy on {status}" + ); + } + } + + #[tokio::test] + async fn inactive_ad_stack_preserves_gpt_diagnostics_cache_privacy() { + // Arrange + let mut settings = settings_with_disabled_ad_templates(); + settings + .integrations + .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) + .expect("should enable GPT diagnostics"); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article?ts_console=1") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build GPT diagnostics request"); + + // Act + let response = run_with_slots(&settings, &services, &[article_slot()], request).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, no-store"), + "active GPT diagnostics should retain cache privacy when server-side ad templates are inactive" + ); + } + + #[tokio::test] + async fn inactive_ad_stack_preserves_non_get_and_non_document_cache_policy() { + let settings = settings_with_disabled_ad_templates(); + + for request in [ + HttpRequest::builder() + .method(Method::POST) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build non-GET document request"), + HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "empty") + .body(EdgeBody::empty()) + .expect("should build non-document request"), + ] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[article_slot()], request).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-cache"), + "inactive server-side ad templates should preserve non-document request policy" ); } } diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index b763cac24..c745fdb2f 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1350,9 +1350,12 @@ unit it maps to (`gam_unit_path`). `enabled` is the dedicated server-side ad-template switch. It defaults to `true` for compatibility with existing configurations. Set it to `false` to stop publisher HTML and SPA page-bids template delivery while retaining the slot -configuration and direct `POST /auction` endpoint. The browser-facing cache -policy for a disabled template stack is `Cache-Control: max-age=60`, unless the -origin already sends `private` or `no-store`. +configuration and direct `POST /auction` endpoint. For a successful GET +publisher document with an inactive template stack, Trusted Server intentionally +sets the browser-facing policy to `Cache-Control: max-age=60`, replacing the +origin `Cache-Control` value as specified by +[#1007](https://github.com/IABTechLab/trusted-server/issues/1007). Error +responses and non-document requests retain the origin policy. ```toml [creative_opportunities] @@ -1382,6 +1385,13 @@ loader: TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false ``` +> [!WARNING] +> Setting `enabled = false` writes this field into the pushed configuration blob. +> Binaries released before this setting reject the unknown field and fail to load +> settings, which makes every request fail. Before rolling back to an older binary, +> restore `enabled` to its default, re-push and finalize the configuration, then +> roll back the binary. + ### `gam_unit_path` templating `gam_unit_path` is a template. A publisher whose ad unit varies by site section diff --git a/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md index f96bdcb28..9fef46451 100644 --- a/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md +++ b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md @@ -7,8 +7,10 @@ retaining the browser-facing cache policy from issue #1007: - Server-side ad templates active: `Cache-Control: private, no-store`. -- Server-side ad templates inactive: `Cache-Control: max-age=60`, unless the - origin already sends `private` or `no-store`. +- Server-side ad templates inactive: successful GET document HTML uses exactly + `Cache-Control: max-age=60`, replacing the origin browser cache policy. +- Non-200, non-GET, and non-document responses keep the origin browser cache + policy. - CDN-specific cache headers must not change when templates are inactive. **Issue context:** The current cache-policy change uses the runtime @@ -53,15 +55,16 @@ consistent with the current issue #952 behavior: - Remove `ETag` and `Last-Modified`. - Remove `Surrogate-Control`, `Fastly-Surrogate-Control`, `CDN-Cache-Control`, and `Cloudflare-CDN-Cache-Control`. -2. For HTML where the server-side ad stack does not run, including an explicit - template disable: - - Read the browser-facing `Cache-Control` header. - - If its value contains `private` or `no-store`, case-insensitively, preserve - the origin value exactly. - - Otherwise set exactly `Cache-Control: max-age=60`. +2. For a `200 OK` GET document HTML response where the server-side ad stack + does not run, including an explicit template disable: + - Set exactly `Cache-Control: max-age=60`, intentionally replacing any origin + browser cache policy as specified by issue #1007. - Leave validators and all CDN-specific cache headers untouched. -3. Preserve the later adapter response-privacy finalization for cookie-bearing - responses; this plan does not refactor that behavior. +3. For non-200, non-GET, and non-document responses, preserve the origin browser + cache policy. +4. Apply request-scoped privacy finalization after this policy so GPT diagnostics + and cookie-bearing responses can still require `private, no-store`; this plan + does not otherwise refactor that behavior. ## File map @@ -172,11 +175,13 @@ semantics: - [ ] Keep the active-SSAT `private, no-store` behavior and validator/CDN header removal unchanged. - [ ] Keep the inactive-HTML `max-age=60` behavior from issue #1007. -- [ ] Verify that explicit template disable changes only browser-facing - `Cache-Control` for cacheable HTML; preserve `ETag`, `Last-Modified`, and - every CDN-specific header. -- [ ] Verify that origin `private`, `PRIVATE`, `no-store`, and `No-Store` values - remain unchanged. +- [ ] Verify that explicit template disable replaces the browser-facing + `Cache-Control` for `200 OK` GET document HTML, including origin + `private`, `no-store`, `no-cache`, and zero-age policies. +- [ ] Preserve `ETag`, `Last-Modified`, and every CDN-specific header. +- [ ] Verify that non-200, non-GET, and non-document responses retain the origin + browser cache policy. +- [ ] Verify that request-scoped GPT diagnostics privacy overrides this policy. ### Task 4: Gate SPA page-bids/template delivery diff --git a/trusted-server.example.toml b/trusted-server.example.toml index ed20c4083..ae1eebdd1 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -182,7 +182,9 @@ auction_html_comment = false [creative_opportunities] # Set to false to disable server-side ad templates while retaining slot definitions -# and direct POST /auction callers. +# and direct POST /auction callers. This intentionally sets successful GET publisher +# documents to Cache-Control: max-age=60. Before rolling back to a binary that +# predates this setting, restore true, re-push and finalize the config first. enabled = true gam_network_id = "123456789" # FCP is not affected by this value — body content above has already From 38c963694f2c915b8380be244c67ee6401555210 Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 17 Aug 2026 12:56:59 -0500 Subject: [PATCH 290/395] Preserve origin cache privacy on inactive templates --- CHANGELOG.md | 2 +- .../src/middleware.rs | 10 +- .../tests/config_env_overlay.rs | 50 +++++- crates/trusted-server-core/src/publisher.rs | 170 ++++++++++++++++-- crates/trusted-server-core/src/settings.rs | 4 +- docs/guide/configuration.md | 31 +++- ...-server-side-ad-templates-cache-control.md | 41 +++-- trusted-server.example.toml | 8 +- 8 files changed, 264 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a221f7af5..62f9dfd0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Publisher HTML uses `Cache-Control: max-age=60` for successful GET document responses when server-side ad templates are inactive, intentionally replacing the origin browser cache policy while preserving CDN-specific cache headers. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers. An explicit `enabled = false` is not compatible with older binaries: restore the default, re-push and finalize the config before rolling back. +- Publisher HTML uses `Cache-Control: max-age=60` for successful GET document responses when server-side ad templates are structurally inactive, while preserving origin `private`/`no-store` policies and request-scoped bot, prefetch, or consent-denied responses. Cookie-bearing responses are finalized as `private, max-age=0`; CDN-specific cache headers remain unchanged for inactive templates. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers; an absent configuration, an unmatched slot, or a disabled auction also make the stack structurally inactive. An explicit `enabled = false` is not compatible with older binaries: restore the default, re-push and finalize the config before rolling back. - **Breaking** — Replaced the legacy APS contextual integration with APS OpenRTB at `/e/pb/bid`. APS configuration now uses canonical `account_id` (`pub_id` remains a compatibility alias), no longer requires APS-specific slot IDs, and defaults script creative eligibility off. Operators must update the endpoint, disable native APS demand for Trusted Server cohorts, and prepare GAM/Universal Creative targeting for `hb_bidder=aps` before rollout. `aps` entries in Prebid bidder lists are logged and stripped. APS renderer winners now preserve the upstream bid `id`, omit `crid` when APS omits it, and carry `ext.trusted_server.renderer` instead of `adm`; external `/auction` consumers must support this response shape. - **Breaking** — All auction paths now forward only a validated publisher-owned page URL as `site.page`, removing query and fragment data. APS OpenRTB omits `site.ref`; the existing Prebid Server path continues to forward the browser `Referer` as `site.ref`. Query-driven sites may lose contextual targeting and per-page reporting signals that previously came from query parameters. - **Breaking** — `bid_param_zone_overrides` inner values must now be JSON objects; previously non-object or empty values (`"header" = "x"`, `"header" = {}`) were accepted and silently produced a dead rule at runtime. They now fail at startup with a configuration error. Operators upgrading should audit their `bid_param_zone_overrides` config for non-object zone entries. diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 2c00ac2ff..a794ee178 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -429,12 +429,12 @@ mod tests { } #[test] - fn enforce_set_cookie_cache_privacy_downgrades_late_cookie() { + fn enforce_set_cookie_cache_privacy_downgrades_inactive_cache_policy() { // Mirrors the EdgeZero post-ec_finalize guard: a Set-Cookie added after - // finalize headers ran (origin-public response) must be downgraded. + // finalize headers ran must override the inactive template cache policy. let mut response = response_with_headers(&[ ("set-cookie", "ts-ec=abc; Path=/"), - ("cache-control", "public, max-age=600"), + ("cache-control", "max-age=60"), ("surrogate-control", "max-age=600"), ]); @@ -446,11 +446,11 @@ mod tests { .get("cache-control") .and_then(|v| v.to_str().ok()), Some("private, max-age=0"), - "should downgrade a late public cookie response to private" + "should downgrade an inactive cache policy on a cookie response" ); assert!( response.headers().get("surrogate-control").is_none(), - "should strip surrogate-control from the late cookie response" + "should strip surrogate-control from the inactive cookie response" ); } diff --git a/crates/trusted-server-cli/tests/config_env_overlay.rs b/crates/trusted-server-cli/tests/config_env_overlay.rs index 39345137b..7d3c38260 100644 --- a/crates/trusted-server-cli/tests/config_env_overlay.rs +++ b/crates/trusted-server-cli/tests/config_env_overlay.rs @@ -29,6 +29,7 @@ ids = ["trusted_server_secrets"] "#; const REWRITE_ENV: &str = "TRUSTED_SERVER__AUCTION__REWRITE_CREATIVES"; const SANITIZE_ENV: &str = "TRUSTED_SERVER__AUCTION__SANITIZE_CREATIVES"; +const AD_TEMPLATES_ENABLED_ENV: &str = "TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED"; struct MigratedProject { directory: TempDir, @@ -44,10 +45,12 @@ fn migrated_legacy_project() -> MigratedProject { .parse::() .expect("should parse legacy integration config"); // EdgeZero v0.0.4 environment overlays cannot create missing TOML leaves, - // so a migrated config must carry both creative-processing leaves for the - // corresponding environment variables to take effect. + // so a migrated config must carry every leaf whose environment override is + // expected to take effect. document["auction"]["rewrite_creatives"] = value(true); document["auction"]["sanitize_creatives"] = value(false); + document["creative_opportunities"]["enabled"] = value(true); + document["creative_opportunities"]["gam_network_id"] = value("123456789"); fs::write(&config_path, document.to_string()).expect("should write migrated config"); fs::write(&manifest_path, MANIFEST).expect("should write test manifest"); MigratedProject { @@ -112,6 +115,49 @@ fn migrated_legacy_config_applies_rewrite_creatives_environment_override() { ); } +#[test] +fn migrated_legacy_config_applies_creative_opportunities_enabled_environment_override() { + let project = migrated_legacy_project(); + let output = Command::new(env!("CARGO_BIN_EXE_ts")) + .args(["config", "push", "--adapter", "axum", "--manifest"]) + .arg(&project.manifest_path) + .arg("--app-config") + .arg(&project.config_path) + .args(["--yes", "--no-diff"]) + .current_dir(project.directory.path()) + .env(AD_TEMPLATES_ENABLED_ENV, "false") + .output() + .expect("should run ts config push"); + + assert!( + output.status.success(), + "valid boolean overlay should push successfully: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let local_store_path = project + .directory + .path() + .join(".edgezero/local-config-trusted_server_config.json"); + let local_store: serde_json::Value = serde_json::from_str( + &fs::read_to_string(local_store_path).expect("should read pushed local config"), + ) + .expect("should parse local config store"); + let envelope_json = local_store + .as_object() + .and_then(|entries| entries.values().next()) + .and_then(serde_json::Value::as_str) + .expect("should contain a blob envelope"); + let envelope: serde_json::Value = + serde_json::from_str(envelope_json).expect("should parse blob envelope"); + + assert_eq!( + envelope["data"]["creative_opportunities"]["enabled"], + serde_json::Value::Bool(false), + "pushed config should contain the environment override" + ); +} + #[test] fn migrated_legacy_config_applies_sanitize_creatives_environment_override() { let project = migrated_legacy_project(); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 055cb81de..780ca3e4b 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -5000,14 +5000,40 @@ mod tests { ); } + fn non_regulated_consent() -> crate::consent::ConsentContext { + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + } + } + async fn run_with_slots( settings: &Settings, services: &RuntimeServices, slots: &[CreativeOpportunitySlot], req: Request, + ) -> PublisherResponse { + run_with_slots_and_consent(settings, services, slots, req, non_regulated_consent()) + .await + } + + async fn run_with_slots_and_consent( + settings: &Settings, + services: &RuntimeServices, + slots: &[CreativeOpportunitySlot], + req: Request, + consent: crate::consent::ConsentContext, ) -> PublisherResponse { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - run_with_orchestrator(settings, services, &orchestrator, slots, req).await + run_with_orchestrator_and_consent( + settings, + services, + &orchestrator, + slots, + req, + consent, + ) + .await } async fn run_with_orchestrator( @@ -5017,10 +5043,25 @@ mod tests { slots: &[CreativeOpportunitySlot], req: Request, ) -> PublisherResponse { - let consent = crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }; + run_with_orchestrator_and_consent( + settings, + services, + orchestrator, + slots, + req, + non_regulated_consent(), + ) + .await + } + + async fn run_with_orchestrator_and_consent( + settings: &Settings, + services: &RuntimeServices, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + consent: crate::consent::ConsentContext, + ) -> PublisherResponse { let mut ec_context = EcContext::new_for_test(None, consent); handle_publisher_request( @@ -5303,14 +5344,7 @@ mod tests { async fn navigation_without_matched_slots_replaces_origin_cache_policy() { let settings = settings_with_enabled_auction_and_creative_opportunities(); - for cache_control in [ - "no-cache", - "max-age=0", - "must-revalidate", - "s-maxage=0", - "private, max-age=0", - "No-Store", - ] { + for cache_control in ["no-cache", "max-age=0", "must-revalidate", "s-maxage=0"] { // Arrange let stub = Arc::new(StubHttpClient::new()); queue_html_response_with_cache_control(&stub, cache_control); @@ -5336,6 +5370,116 @@ mod tests { } } + #[tokio::test] + async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "inactive server-side ad templates should preserve private origin {cache_control} policy" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "inactive server-side ad templates should preserve {header_name}" + ); + } + } + } + + #[tokio::test] + async fn request_scoped_ad_stack_suppression_preserves_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let slots = [article_slot()]; + let mut bot_request = conditional_navigation_request(); + bot_request.headers_mut().insert( + "user-agent", + HeaderValue::from_static("Mozilla/5.0 (compatible; Googlebot/2.1)"), + ); + let mut prefetch_request = conditional_navigation_request(); + prefetch_request + .headers_mut() + .insert("sec-purpose", HeaderValue::from_static("prefetch")); + + for (skip_reason, request, consent) in [ + ("bot", bot_request, non_regulated_consent()), + ("prefetch", prefetch_request, non_regulated_consent()), + ( + "consent denied", + conditional_navigation_request(), + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots_and_consent(&settings, &services, &slots, request, consent) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-cache"), + "{skip_reason} should retain the origin cache policy" + ); + } + } + #[tokio::test] async fn absent_creative_opportunities_use_short_browser_cache_policy() { // Arrange diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 81a2323ea..c6e93e688 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -5042,7 +5042,7 @@ formats = [{ width = 300, height = 250 }] } #[test] - fn settings_creative_opportunity_enabled_flag_supports_environment_override() { + fn legacy_settings_loader_applies_creative_opportunity_enabled_environment_override() { let toml = format!( "{}\n[creative_opportunities]\nenabled = true\ngam_network_id = \"21765378893\"\n", crate_test_settings_str() @@ -5062,7 +5062,7 @@ formats = [{ width = 300, height = 250 }] .creative_opportunities .expect("should have creative opportunities") .enabled, - "environment override should disable template delivery" + "legacy settings loader should disable template delivery" ); }); } diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index c745fdb2f..06c671b83 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1350,12 +1350,24 @@ unit it maps to (`gam_unit_path`). `enabled` is the dedicated server-side ad-template switch. It defaults to `true` for compatibility with existing configurations. Set it to `false` to stop publisher HTML and SPA page-bids template delivery while retaining the slot -configuration and direct `POST /auction` endpoint. For a successful GET -publisher document with an inactive template stack, Trusted Server intentionally -sets the browser-facing policy to `Cache-Control: max-age=60`, replacing the -origin `Cache-Control` value as specified by -[#1007](https://github.com/IABTechLab/trusted-server/issues/1007). Error -responses and non-document requests retain the origin policy. +configuration and direct `POST /auction` endpoint. + +#### Publisher document cache policy + +For a successful GET publisher document, Trusted Server applies the +browser-facing `Cache-Control: max-age=60` policy from +[#1007](https://github.com/IABTechLab/trusted-server/issues/1007) when the +server-side ad stack is structurally inactive. This includes an absent +`[creative_opportunities]` section, `enabled = false`, no slot matching the +path, or a disabled auction. The policy replaces the origin browser cache +policy except when the origin sends `private` or `no-store`, which are +preserved. Bot, prefetch, and consent-denied requests also retain the origin +policy because they can produce a request-specific representation for the same +URL. Error responses and non-document requests retain the origin policy. + +Any response that carries `Set-Cookie` is finalized as +`Cache-Control: private, max-age=0`; this privacy rule takes precedence over +the short inactive-stack policy. ```toml [creative_opportunities] @@ -1378,8 +1390,11 @@ page_patterns = ["/", "/news", "/news/*", "/reviews", "/reviews/*"] formats = [{ width = 728, height = 90 }] ``` -The same switch can be overridden through the legacy environment-variable -loader: +The same switch can be overridden through the typed CLI environment overlay. +Because EdgeZero only replaces TOML leaves that already exist, first add +`enabled = true` to the `[creative_opportunities]` block in the base config +before using this override. See [Environment Variable Overrides (Typed +CLI)](#environment-variable-overrides-typed-cli) for the general overlay rules. ```bash TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false diff --git a/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md index 9fef46451..b950d0a95 100644 --- a/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md +++ b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md @@ -7,10 +7,11 @@ retaining the browser-facing cache policy from issue #1007: - Server-side ad templates active: `Cache-Control: private, no-store`. -- Server-side ad templates inactive: successful GET document HTML uses exactly - `Cache-Control: max-age=60`, replacing the origin browser cache policy. -- Non-200, non-GET, and non-document responses keep the origin browser cache - policy. +- Structurally inactive server-side ad templates: successful request-eligible GET + document HTML uses exactly `Cache-Control: max-age=60`, except origin + `private`/`no-store` policies remain unchanged. +- Non-200, non-GET, non-document, bot, prefetch, and consent-denied responses + keep the origin browser cache policy. - CDN-specific cache headers must not change when templates are inactive. **Issue context:** The current cache-policy change uses the runtime @@ -55,16 +56,18 @@ consistent with the current issue #952 behavior: - Remove `ETag` and `Last-Modified`. - Remove `Surrogate-Control`, `Fastly-Surrogate-Control`, `CDN-Cache-Control`, and `Cloudflare-CDN-Cache-Control`. -2. For a `200 OK` GET document HTML response where the server-side ad stack - does not run, including an explicit template disable: - - Set exactly `Cache-Control: max-age=60`, intentionally replacing any origin - browser cache policy as specified by issue #1007. +2. For a request-eligible `200 OK` GET document HTML response where the + server-side ad stack is structurally inactive, including an explicit template + disable: + - Set exactly `Cache-Control: max-age=60`, replacing origin browser policies + as specified by issue #1007 unless the origin sends `private` or `no-store`. - Leave validators and all CDN-specific cache headers untouched. -3. For non-200, non-GET, and non-document responses, preserve the origin browser - cache policy. +3. For non-200, non-GET, non-document, bot, prefetch, and consent-denied + responses, preserve the origin browser cache policy. 4. Apply request-scoped privacy finalization after this policy so GPT diagnostics - and cookie-bearing responses can still require `private, no-store`; this plan - does not otherwise refactor that behavior. + and cookie-bearing responses can still require `private, no-store`; a + cookie-bearing response therefore ends as `private, max-age=0` when it did + not already carry a stricter policy. ## File map @@ -175,13 +178,15 @@ semantics: - [ ] Keep the active-SSAT `private, no-store` behavior and validator/CDN header removal unchanged. - [ ] Keep the inactive-HTML `max-age=60` behavior from issue #1007. -- [ ] Verify that explicit template disable replaces the browser-facing - `Cache-Control` for `200 OK` GET document HTML, including origin - `private`, `no-store`, `no-cache`, and zero-age policies. +- [ ] Verify that structurally inactive request-eligible responses replace the + browser-facing `Cache-Control` for `200 OK` GET document HTML, including + origin `no-cache` and zero-age policies, while preserving `private` and + `no-store`. - [ ] Preserve `ETag`, `Last-Modified`, and every CDN-specific header. -- [ ] Verify that non-200, non-GET, and non-document responses retain the origin - browser cache policy. -- [ ] Verify that request-scoped GPT diagnostics privacy overrides this policy. +- [ ] Verify that non-200, non-GET, non-document, bot, prefetch, and + consent-denied responses retain the origin browser cache policy. +- [ ] Verify that request-scoped GPT diagnostics and cookie privacy override this + policy. ### Task 4: Gate SPA page-bids/template delivery diff --git a/trusted-server.example.toml b/trusted-server.example.toml index ae1eebdd1..ce5cd3e59 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -182,9 +182,11 @@ auction_html_comment = false [creative_opportunities] # Set to false to disable server-side ad templates while retaining slot definitions -# and direct POST /auction callers. This intentionally sets successful GET publisher -# documents to Cache-Control: max-age=60. Before rolling back to a binary that -# predates this setting, restore true, re-push and finalize the config first. +# and direct POST /auction callers. Structurally inactive templates use +# Cache-Control: max-age=60; this setting is one cause. Origin private/no-store +# policies and bot, prefetch, or consent-denied requests retain their origin policy. +# Before rolling back to a binary that predates this setting, restore true, re-push +# and finalize the config first. enabled = true gam_network_id = "123456789" # FCP is not affected by this value — body content above has already From fd6d832488126f08b9b418b7fe35056a8cfc999b Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 17 Aug 2026 13:27:56 -0500 Subject: [PATCH 291/395] Resolve cache policy review feedback --- .../wrangler.ci.toml | 3 - .../wrangler.toml | 3 - .../trusted-server-core/src/cache_policy.rs | 32 +- .../src/integrations/registry.rs | 7 + crates/trusted-server-core/src/publisher.rs | 449 +++++++++++++++--- .../src/response_privacy.rs | 29 ++ crates/trusted-server-core/src/settings.rs | 72 ++- .../tests/common/ec.rs | 76 +++ .../tests/integration.rs | 50 ++ docs/guide/configuration.md | 61 +-- 10 files changed, 672 insertions(+), 110 deletions(-) diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml index 88a2dfc74..e6891eb79 100644 --- a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml +++ b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml @@ -6,9 +6,6 @@ compatibility_date = "2024-09-23" compatibility_flags = ["nodejs_compat", "cache_option_enabled"] # No [build] section — bundle is pre-built in CI; wrangler dev must not rebuild. -[cache] -enabled = true - [[kv_namespaces]] binding = "TRUSTED_SERVER_KV" id = "ci-local-kv" diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.toml b/crates/trusted-server-adapter-cloudflare/wrangler.toml index 9acdb13ab..7c91173fc 100644 --- a/crates/trusted-server-adapter-cloudflare/wrangler.toml +++ b/crates/trusted-server-adapter-cloudflare/wrangler.toml @@ -11,9 +11,6 @@ compatibility_date = "2024-09-23" # (auction-eligible publisher navigations), so this is a hard requirement. compatibility_flags = ["nodejs_compat", "cache_option_enabled"] -[cache] -enabled = true - [build] command = "bash build.sh" diff --git a/crates/trusted-server-core/src/cache_policy.rs b/crates/trusted-server-core/src/cache_policy.rs index 39bac467c..4d920f54a 100644 --- a/crates/trusted-server-core/src/cache_policy.rs +++ b/crates/trusted-server-core/src/cache_policy.rs @@ -289,14 +289,38 @@ pub fn is_edge_cache_header_name(name: &str) -> bool { /// as `not-private` or `no-storey` do not match `private` / `no-store`. #[must_use] pub fn cache_control_value_has_directive(value: &str, directive: &str) -> bool { - value.split(',').any(|part| { + let part_has_directive = |part: &str| { let part = part.trim(); let directive_name = part .find(['=', ';']) .map_or(part, |end| &part[..end]) .trim(); directive_name.eq_ignore_ascii_case(directive) - }) + }; + + let mut quoted = false; + let mut escaped = false; + let mut part_start = 0; + for (index, character) in value.char_indices() { + if quoted { + if escaped { + escaped = false; + } else if character == '\\' { + escaped = true; + } else if character == '"' { + quoted = false; + } + } else if character == '"' { + quoted = true; + } else if character == ',' { + if part_has_directive(&value[part_start..index]) { + return true; + } + part_start = index + character.len_utf8(); + } + } + + part_has_directive(&value[part_start..]) } /// Return true when any `Cache-Control` header value contains `directive`. @@ -544,6 +568,10 @@ mod tests { !cache_control_value_has_directive("public, no-storey, not-private", "private"), "should not match pseudo-private directives by substring" ); + assert!( + !cache_control_value_has_directive("public, ext=\"a,no-store,b\"", "no-store"), + "should ignore directive-shaped text inside quoted extension values" + ); } #[test] diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 16cbac868..ed7970eaf 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1134,6 +1134,13 @@ impl IntegrationRegistry { ids } + /// Return whether an integration is enabled, including integrations whose + /// JavaScript is delivered outside the standard module bundles. + #[must_use] + pub fn is_enabled(&self, integration_id: &str) -> bool { + self.inner.enabled_integration_ids.contains(&integration_id) + } + /// Return JS module IDs for the main (synchronous) bundle, excluding /// modules registered with [`with_deferred_js`](IntegrationRegistrationBuilder::with_deferred_js). #[must_use] diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index df57a3450..6e94f16f6 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -277,11 +277,13 @@ fn accept_encoding_qvalue(header_value: &str, target: &str) -> Option { /// Unified tsjs static serving: `/static/tsjs=` /// -/// Serves two types of bundles: +/// Serves three types of bundles: /// - **Unified bundle** (`tsjs-unified.min.js`): core + immediate (non-deferred) /// integration modules. /// - **Deferred module** (`tsjs-{id}.min.js`): a single self-contained IIFE for -/// modules loaded with `defer` (e.g., prebid). +/// modules loaded with `defer` (e.g., Prebid). +/// - **Standalone diagnostics module** (`tsjs-gpt_diagnostics.min.js`): delivered +/// only when the diagnostics integration is enabled and a document activates it. /// /// # Errors /// @@ -309,9 +311,11 @@ pub fn handle_tsjs_dynamic( } if let Some(module_id) = parse_deferred_module_filename(filename) { - // Only serve if the deferred module is actually enabled let deferred_ids = integration_registry.js_module_ids_deferred(); - if !deferred_ids.contains(&module_id) { + let is_enabled_diagnostics_module = module_id + == crate::integrations::gpt_diagnostics::GPT_DIAGNOSTICS_INTEGRATION_ID + && integration_registry.is_enabled(module_id); + if !deferred_ids.contains(&module_id) && !is_enabled_diagnostics_module { return Ok(not_found_response()); } if let (Some(content), Some(hash)) = ( @@ -381,6 +385,9 @@ struct ProcessResponseParams<'a> { integration_registry: &'a IntegrationRegistry, ad_slots_script: Option<&'a str>, ad_bids_state: &'a Arc>>, + suppress_datadome_client_side_tag: bool, + gpt_diagnostics: + Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, } struct PublisherBodyProcessor { @@ -397,15 +404,17 @@ impl PublisherBodyProcessor { let is_rsc_flight = content_type_contains_ascii_case_insensitive(¶ms.content_type, "text/x-component"); let inner: Box = if is_html { - Box::new(create_html_stream_processor( - ¶ms.origin_host, - ¶ms.request_host, - ¶ms.request_scheme, + Box::new(create_html_stream_processor(HtmlStreamProcessorParams { + origin_host: ¶ms.origin_host, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, settings, integration_registry, - params.ad_slots_script.as_deref().map(str::to_string), - Arc::clone(¶ms.ad_bids_state), - )?) + ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), + ad_bids_state: Arc::clone(¶ms.ad_bids_state), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, + gpt_diagnostics: params.gpt_diagnostics.clone(), + })?) } else if is_rsc_flight { Box::new(RscFlightUrlRewriter::new( ¶ms.origin_host, @@ -473,15 +482,17 @@ fn process_response_streaming( let max_pending_decoded_bytes = params.settings.publisher.max_buffered_body_bytes; if is_html { - let processor = create_html_stream_processor( - params.origin_host, - params.request_host, - params.request_scheme, - params.settings, - params.integration_registry, - params.ad_slots_script.map(str::to_string), - params.ad_bids_state.clone(), - )?; + let processor = create_html_stream_processor(HtmlStreamProcessorParams { + origin_host: params.origin_host, + request_host: params.request_host, + request_scheme: params.request_scheme, + settings: params.settings, + integration_registry: params.integration_registry, + ad_slots_script: params.ad_slots_script.map(str::to_string), + ad_bids_state: params.ad_bids_state.clone(), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, + gpt_diagnostics: params.gpt_diagnostics.cloned(), + })?; StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) .process(body_as_reader(body)?, output)?; @@ -956,25 +967,33 @@ async fn hold_finish_tail_segments( /// `use<>` states that explicitly: without it, Rust 2024 would have the opaque /// type capture every input lifetime, forcing callers to keep the settings and /// registry alive for as long as the processor. -fn create_html_stream_processor( - origin_host: &str, - request_host: &str, - request_scheme: &str, - settings: &Settings, - integration_registry: &IntegrationRegistry, +struct HtmlStreamProcessorParams<'a> { + origin_host: &'a str, + request_host: &'a str, + request_scheme: &'a str, + settings: &'a Settings, + integration_registry: &'a IntegrationRegistry, ad_slots_script: Option, ad_bids_state: Arc>>, + suppress_datadome_client_side_tag: bool, + gpt_diagnostics: Option, +} + +fn create_html_stream_processor( + params: HtmlStreamProcessorParams<'_>, ) -> Result, Report> { use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; let config = HtmlProcessorConfig::from_settings( - settings, - integration_registry, - origin_host, - request_host, - request_scheme, + params.settings, + params.integration_registry, + params.origin_host, + params.request_host, + params.request_scheme, ) - .with_ad_state(ad_slots_script, ad_bids_state); + .with_ad_state(params.ad_slots_script, params.ad_bids_state) + .with_gpt_diagnostics(params.gpt_diagnostics) + .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); Ok(create_html_processor(config)) } @@ -1123,6 +1142,11 @@ pub struct OwnedProcessResponseParams { pub(crate) dispatched_auction: Option, /// Price granularity used to bucket bids when building `tsjs.bids`. pub(crate) price_granularity: PriceGranularity, + /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. + pub(crate) suppress_datadome_client_side_tag: bool, + /// Request-scoped conditional diagnostics delivery decision. + pub(crate) gpt_diagnostics: + Option, } /// Buffers a [`PublisherResponse`] into a single [`Response`], collecting the @@ -1455,6 +1479,45 @@ pub async fn publisher_response_into_streaming_response( } } +/// Returns whether a request can render an HTML document context. +fn is_html_document_request(req: &Request) -> bool { + if let Some(destination) = req + .headers() + .get("sec-fetch-dest") + .and_then(|value| value.to_str().ok()) + { + return matches!( + destination.trim().to_ascii_lowercase().as_str(), + "document" | "embed" | "fencedframe" | "frame" | "iframe" | "object" + ); + } + + is_navigation_request(req) +} + +/// Removes request headers that can produce a bodyless or partial origin response. +fn strip_conditional_and_range_headers(req: &mut Request) { + req.headers_mut().remove(header::IF_NONE_MATCH); + req.headers_mut().remove(header::IF_MODIFIED_SINCE); + req.headers_mut().remove(header::RANGE); + req.headers_mut().remove(header::IF_RANGE); +} + +/// Prevent shared caches from replaying tag-suppressed HTML to other clients. +fn apply_datadome_client_tag_cache_privacy( + response: &mut Response, + method: &Method, + suppress_datadome_client_side_tag: bool, + content_type: &str, +) { + if suppress_datadome_client_side_tag + && response_carries_body(method, response.status()) + && is_html_content_type(content_type) + { + enforce_synthesized_html_cache_privacy(response); + } +} + /// Returns `true` when a buffered publisher response should carry a body and a /// recomputed `Content-Length`. /// @@ -1586,6 +1649,8 @@ pub fn stream_publisher_body( integration_registry, ad_slots_script: params.ad_slots_script.as_deref(), ad_bids_state: ¶ms.ad_bids_state, + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, + gpt_diagnostics: params.gpt_diagnostics.as_ref(), }; process_response_streaming(body, output, &borrowed) } @@ -1670,15 +1735,17 @@ pub async fn stream_publisher_body_async( // HTML: build the processor once and drive it chunk by chunk. // One-behind buffer: stream chunk N-1 immediately; hold chunk N until origin // EOF, then await auction and process chunk N (which contains ). - let mut processor = match create_html_stream_processor( - ¶ms.origin_host, - ¶ms.request_host, - ¶ms.request_scheme, + let mut processor = match create_html_stream_processor(HtmlStreamProcessorParams { + origin_host: ¶ms.origin_host, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, settings, integration_registry, - params.ad_slots_script.as_deref().map(str::to_string), - params.ad_bids_state.clone(), - ) { + ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), + ad_bids_state: params.ad_bids_state.clone(), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, + gpt_diagnostics: params.gpt_diagnostics.clone(), + }) { Ok(processor) => processor, Err(err) => { emit_abandoned_auction( @@ -2447,6 +2514,7 @@ async fn collect_non_html_auction( AuctionTerminalOutcome::Completed { request: auction_request, result: &result, + delivered_winner_slots: None, }, ) }) @@ -2493,6 +2561,7 @@ async fn collect_stream_auction( AuctionTerminalOutcome::Completed { request: auction_request, result: &result, + delivered_winner_slots: None, }, ) }) @@ -2580,6 +2649,11 @@ pub async fn handle_publisher_request( ) -> Result> { log::debug!("Proxying request to publisher_origin"); + // Adapter fallbacks prepare this before EC/cookie handling. Keep this + // idempotent call as a direct-handler safety net and for focused tests. + let gpt_diagnostics = + crate::integrations::gpt_diagnostics::prepare_request(settings, &mut req)?; + // Prebid.js requests are not intercepted here anymore. The HTML processor removes // publisher-supplied Prebid scripts; the unified TSJS bundle includes Prebid.js when enabled. @@ -2666,11 +2740,13 @@ pub async fn handle_publisher_request( let is_prefetch = is_prefetch_request(&req); let is_bot = is_bot_user_agent(&req); - let matched_slots: Vec<_> = if settings.creative_opportunities.is_some() && is_get { - crate::creative_opportunities::match_slots(auction.slots, &request_path) - .into_iter() - .cloned() - .collect() + let matched_slots = if is_get { + settings + .creative_opportunities + .as_ref() + .map_or_else(Vec::new, |co_config| { + match_renderable_slots(auction.slots, co_config, &request_path) + }) } else { Vec::new() }; @@ -2861,6 +2937,18 @@ pub async fn handle_publisher_request( } ); + let suppress_datadome_client_side_tag = req + .extensions() + .get::() + .is_some(); + if should_run_ad_stack || (suppress_datadome_client_side_tag && is_html_document_request(&req)) + { + // HTML document contexts whose output may be synthesized must not + // receive a cached 304 or partial 206. Non-document subresources contain + // no executable injected tag, so retain their validators and ranges. + strip_conditional_and_range_headers(&mut req); + } + // Only advertise encodings the rewrite pipeline can decode and re-encode. restrict_accept_encoding(&mut req); // Strip the internal `fastly-ssl` scheme signal before forwarding to the @@ -2888,6 +2976,9 @@ pub async fn handle_publisher_request( if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); } + if should_run_ad_stack { + platform_request = platform_request.with_cache_bypass(); + } let mut response = match services.http_client().send(platform_request).await { Ok(platform_response) => platform_response.response, @@ -2913,11 +3004,37 @@ pub async fn handle_publisher_request( response.headers().len() ); + if should_run_ad_stack && response.status() == StatusCode::NOT_MODIFIED { + if let Some(dispatched) = dispatched_auction.take() { + emit_abandoned_auction( + services, + auction_observation.take(), + dispatched, + "unexpected_origin_304", + ) + .await; + } + + let response = Response::builder() + .status(StatusCode::BAD_GATEWAY) + .header(header::CACHE_CONTROL, "private, no-store") + .header(header::CONTENT_TYPE, "text/plain; charset=utf-8") + .body(EdgeBody::from( + "Publisher origin returned an invalid conditional response", + )) + .change_context(TrustedServerError::Proxy { + message: "failed to build unexpected origin 304 response".to_string(), + })?; + return Ok(PublisherResponse::Buffered(response)); + } + + crate::integrations::gpt_diagnostics::finalize_response(&gpt_diagnostics, &mut response); + let ad_slots_script = if should_run_ad_stack { settings .creative_opportunities .as_ref() - .map(|co_config| build_ad_slots_script(&matched_slots, co_config)) + .map(|co_config| build_ad_slots_script(&matched_slots, co_config, &request_path)) } else { None }; @@ -2937,10 +3054,17 @@ pub async fn handle_publisher_request( .headers() .get(header::CONTENT_TYPE) .and_then(|h| h.to_str().ok()) - .unwrap_or_default(); - if should_run_ad_stack && is_html_content_type(origin_content_type) { + .unwrap_or_default() + .to_string(); + if should_run_ad_stack && is_html_content_type(&origin_content_type) { enforce_synthesized_html_cache_privacy(&mut response); } + apply_datadome_client_tag_cache_privacy( + &mut response, + &request_method, + suppress_datadome_client_side_tag, + &origin_content_type, + ); apply_publisher_asset_cache_policy( settings, @@ -3058,6 +3182,8 @@ pub async fn handle_publisher_request( auction_request: auction_request_for_telemetry, dispatched_auction, price_granularity, + suppress_datadome_client_side_tag, + gpt_diagnostics: Some(gpt_diagnostics), }), }) } @@ -3399,8 +3525,9 @@ pub(crate) fn build_empty_bids_script() -> String { fn build_slot_json( slot: &crate::creative_opportunities::CreativeOpportunitySlot, co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, -) -> serde_json::Value { - let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); + section: &str, +) -> Option { + let gam_path = slot.render_gam_unit_path(&co_config.gam_network_id, section)?; let div_id = slot.resolved_div_id(); let formats: Vec = slot .formats @@ -3412,13 +3539,40 @@ fn build_slot_json( .iter() .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) .collect(); - serde_json::json!({ + Some(serde_json::json!({ "id": slot.id, "gam_unit_path": gam_path, "div_id": div_id, "formats": formats, "targeting": targeting, - }) + })) +} + +/// Match creative-opportunity slots and omit dynamic GAM paths that cannot be +/// rendered for this request before they can enter an auction. +fn match_renderable_slots( + slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, + request_path: &str, +) -> Vec { + let section = co_config.section_for_path(request_path); + crate::creative_opportunities::match_slots(slots, request_path) + .into_iter() + .filter_map(|slot| { + if slot + .render_gam_unit_path(&co_config.gam_network_id, §ion) + .is_none() + { + log::warn!( + "Omitting slot `{}`: dynamic gam_unit_path exceeds the render limit for path `{}`", + slot.id, + request_path + ); + return None; + } + Some(slot.clone()) + }) + .collect() } /// Build the `tsjs.adSlots` `"); @@ -7794,6 +8086,8 @@ mod tests { // as the `/auction` path (sanitize → rewrite) before the creative // reaches window.tsjs.bids, so hostile executable markup never lands // in the client-facing `adm` for the Prebid Universal Creative to run. + let mut settings = test_settings(); + settings.auction.sanitize_creatives = true; let mut winning_bids = HashMap::new(); let mut bid = make_bid( "atf_sidebar_ad", @@ -7810,13 +8104,7 @@ mod tests { ); winning_bids.insert("atf_sidebar_ad".to_string(), bid); - let map = build_bid_map( - &winning_bids, - PriceGranularity::Dense, - &test_settings(), - "", - false, - ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); let adm = map .get("atf_sidebar_ad") .and_then(|v| v.as_object()) @@ -7846,6 +8134,7 @@ mod tests { fn build_bid_map_can_skip_rewriting_but_not_sanitization() { let mut settings = test_settings(); settings.auction.rewrite_creatives = false; + settings.auction.sanitize_creatives = true; let mut winning_bids = HashMap::new(); let mut bid = make_bid( "atf_sidebar_ad", @@ -8235,7 +8524,10 @@ mod tests { height: 250, nurl: None, burl: None, + bid_id: None, ad_id: Some("bid-impression-id".to_string()), + creative_id: None, + renderer: None, cache_id: Some("f47447a0-b759-4f2f-9887-af458b79b570".to_string()), cache_host: Some("openads.adsrvr.org".to_string()), cache_path: Some("/cache".to_string()), @@ -8287,7 +8579,10 @@ mod tests { height: 250, nurl: None, burl: None, + bid_id: None, ad_id: Some("aps-bid-token".to_string()), + creative_id: None, + renderer: None, cache_id: None, cache_host: None, cache_path: None, @@ -8337,7 +8632,10 @@ mod tests { height: 250, nurl: None, burl: None, + bid_id: None, ad_id: None, + creative_id: None, + renderer: None, cache_id: None, cache_host: None, cache_path: None, @@ -8378,7 +8676,10 @@ mod tests { height: 250, nurl: None, burl: None, + bid_id: None, ad_id: None, + creative_id: None, + renderer: None, cache_id: None, cache_host: None, cache_path: None, @@ -8672,6 +8973,7 @@ mod tests { targeting: Default::default(), providers: Default::default(), compiled_patterns: Vec::new(), + compiled_unit: None, }] } @@ -9204,7 +9506,7 @@ mod tests { /// the handler emitted. mod navigation_publisher_domain_tests { use super::*; - use crate::auction::provider::AuctionProvider; + use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; use crate::auction::types::AuctionRequest; use crate::auction::{AuctionContext, AuctionOrchestrator}; @@ -9212,7 +9514,7 @@ mod tests { use crate::platform::test_support::{ NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, }; - use crate::platform::{ClientInfo, PlatformPendingRequest, PlatformResponse}; + use crate::platform::{ClientInfo, PlatformResponse}; use crate::test_support::tests::crate_test_settings_str; use std::sync::Mutex; @@ -9241,7 +9543,7 @@ mod tests { &self, request: &AuctionRequest, _context: &AuctionContext<'_>, - ) -> Result> { + ) -> Result> { *self.captured.lock().expect("should lock captured request") = Some(request.clone()); Err(Report::new(TrustedServerError::Auction { @@ -9314,6 +9616,7 @@ mod tests { targeting: Default::default(), providers: Default::default(), compiled_patterns: Vec::new(), + compiled_unit: None, }] } diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 8ccda97bd..0cc0eb88f 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -290,6 +290,35 @@ mod tests { ); } + #[test] + fn cookie_privacy_ignores_quoted_extension_directives() { + let settings = settings_with_response_headers(&[]); + let mut response = response_builder() + .header(header::SET_COOKIE, "id=abc") + .header( + header::CACHE_CONTROL, + "public, max-age=600, ext=\"a,no-store,b\"", + ) + .header("surrogate-control", "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "quoted extension text must not prevent the cookie privacy downgrade" + ); + assert!( + !response.headers().contains_key("surrogate-control"), + "cookie privacy downgrade should strip edge-cache headers" + ); + } + #[test] fn preserves_private_no_store_against_operator_cache_headers_without_cookie() { let settings = settings_with_response_headers(&[ diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index da61dd84a..cd5f7f8cd 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2056,7 +2056,24 @@ impl CacheAssetRule { } fn validate_policy_shape(&self) -> Result<(), Report> { - if self.browser_ttl_seconds.is_none() && self.edge_ttl_seconds.is_none() { + if self.visibility == CachePolicyVisibility::Private { + if self.edge_ttl_seconds.is_some() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` sets edge_ttl_seconds with private visibility; private rules must use browser_ttl_seconds", + self.id + ), + })); + } + if self.browser_ttl_seconds.is_none() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` with private visibility must configure browser_ttl_seconds", + self.id + ), + })); + } + } else if self.browser_ttl_seconds.is_none() && self.edge_ttl_seconds.is_none() { return Err(Report::new(TrustedServerError::Configuration { message: format!( "cache.asset_rules `{}` must configure browser_ttl_seconds or edge_ttl_seconds", @@ -3481,6 +3498,59 @@ mod tests { format!("{browser_ttl_err:?}").contains("positive browser_ttl_seconds"), "should explain immutable browser TTL requirement: {browser_ttl_err:?}" ); + + let private_edge_only = format!( + r#"{} + + [[cache.asset_rules]] + id = "private-edge-only" + enabled = true + path_prefix = "/assets/" + visibility = "private" + edge_ttl_seconds = 300 + "#, + crate_test_settings_str() + ); + let private_edge_only_err = Settings::from_toml(&private_edge_only) + .expect_err("should reject private rule with only an edge TTL"); + assert!( + format!("{private_edge_only_err:?}").contains("edge_ttl_seconds"), + "should explain that private rules cannot use an edge TTL: {private_edge_only_err:?}" + ); + + let private_dual_ttl = private_edge_only.replace( + "id = \"private-edge-only\"", + "id = \"private-dual-ttl\"\n browser_ttl_seconds = 300", + ); + let private_dual_ttl_err = Settings::from_toml(&private_dual_ttl) + .expect_err("should reject private rule with browser and edge TTLs"); + assert!( + format!("{private_dual_ttl_err:?}").contains("edge_ttl_seconds"), + "should reject edge TTL even when a private rule has a browser TTL: {private_dual_ttl_err:?}" + ); + + let private_browser_ttl = private_edge_only.replace( + "id = \"private-edge-only\"\n enabled = true\n path_prefix = \"/assets/\"\n visibility = \"private\"\n edge_ttl_seconds = 300", + "id = \"private-browser-ttl\"\n enabled = true\n path_prefix = \"/assets/\"\n visibility = \"private\"\n browser_ttl_seconds = 300", + ); + let private_settings = Settings::from_toml(&private_browser_ttl) + .expect("should accept a private rule with a browser TTL"); + let private_policy = private_settings + .asset_cache_policy_for_path("/assets/app.js") + .expect("should evaluate private cache rule") + .expect("should match private cache rule"); + assert_eq!( + private_policy + .cache_control_value(crate::cache_policy::EdgeCacheHeader::SurrogateControl), + "private, max-age=300", + "private rules should render their browser TTL" + ); + assert_eq!( + private_policy + .edge_header_value(crate::cache_policy::EdgeCacheHeader::SurrogateControl), + None, + "private rules should not render an edge cache TTL" + ); } #[test] diff --git a/crates/trusted-server-integration-tests/tests/common/ec.rs b/crates/trusted-server-integration-tests/tests/common/ec.rs index cde6ad1c4..0a1f149c3 100644 --- a/crates/trusted-server-integration-tests/tests/common/ec.rs +++ b/crates/trusted-server-integration-tests/tests/common/ec.rs @@ -403,3 +403,79 @@ impl Drop for MinimalOrigin { } } } + +/// A minimal HTTP origin that reflects the request's Cookie header in a +/// cacheable HTML response. +/// +/// This makes it possible to assert that an edge runtime does not reuse a +/// cookie-influenced publisher response for another visitor. +pub struct CookieVaryingOrigin { + shutdown_tx: mpsc::Sender<()>, + handle: Option>, +} + +impl CookieVaryingOrigin { + /// Starts the cookie-varying origin on `127.0.0.1:{port}`. + /// + /// # Panics + /// + /// Panics if the port is already in use. + pub fn start(port: u16) -> Self { + let listener = + TcpListener::bind(format!("127.0.0.1:{port}")).expect("should bind origin port"); + listener + .set_nonblocking(true) + .expect("should set listener nonblocking"); + let (shutdown_tx, shutdown_rx) = mpsc::channel::<()>(); + + let handle = thread::spawn(move || { + loop { + if shutdown_rx.try_recv().is_ok() { + break; + } + + match listener.accept() { + Ok((mut stream, _addr)) => { + let mut buf = [0u8; 4096]; + let Ok(bytes_read) = stream.read(&mut buf) else { + continue; + }; + let request = String::from_utf8_lossy(&buf[..bytes_read]); + let cookie = request + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("cookie").then(|| value.trim()) + }) + .unwrap_or("viewer=missing"); + let body = format!("{cookie}"); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); + } + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(_) => break, + } + } + }); + + Self { + shutdown_tx, + handle: Some(handle), + } + } +} + +impl Drop for CookieVaryingOrigin { + fn drop(&mut self) { + let _ = self.shutdown_tx.send(()); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} diff --git a/crates/trusted-server-integration-tests/tests/integration.rs b/crates/trusted-server-integration-tests/tests/integration.rs index 76a267d1f..ebd026410 100644 --- a/crates/trusted-server-integration-tests/tests/integration.rs +++ b/crates/trusted-server-integration-tests/tests/integration.rs @@ -2,6 +2,7 @@ mod common; mod environments; mod frameworks; +use common::ec::CookieVaryingOrigin; use common::runtime::{RuntimeEnvironment, TestError, origin_port, wasm_binary_path}; use environments::{RUNTIME_ENVIRONMENTS, ReadyCheckOptions, wait_for_http_ready}; use error_stack::ResultExt as _; @@ -164,6 +165,55 @@ fn test_nextjs_cloudflare() { test_combination(&runtime, &framework).expect("should pass Next.js on Cloudflare Workers"); } +#[test] +#[ignore = "requires the `wrangler` CLI in $PATH and a prebuilt Cloudflare Workers bundle (run build.sh first); the test starts wrangler dev automatically"] +fn test_cloudflare_dynamic_publisher_response_does_not_cross_cookie_boundaries() { + init_logger(); + let _origin = CookieVaryingOrigin::start(origin_port()); + let runtime = environments::cloudflare::CloudflareWorkers; + let process = runtime + .spawn(&wasm_binary_path()) + .expect("should start Cloudflare Worker"); + let client = reqwest::blocking::Client::new(); + + let first_response = client + .get(format!("{}/cache-regression", process.base_url)) + .header("cookie", "viewer=first") + .send() + .expect("should request first dynamic publisher response"); + assert_eq!( + first_response.status().as_u16(), + 200, + "first dynamic publisher response should succeed" + ); + let first_body = first_response + .text() + .expect("should read first dynamic publisher response"); + + let second_response = client + .get(format!("{}/cache-regression", process.base_url)) + .header("cookie", "viewer=second") + .send() + .expect("should request second dynamic publisher response"); + assert_eq!( + second_response.status().as_u16(), + 200, + "second dynamic publisher response should succeed" + ); + let second_body = second_response + .text() + .expect("should read second dynamic publisher response"); + + assert!( + first_body.contains("viewer=first"), + "first response must preserve its cookie-specific origin body: {first_body}" + ); + assert!( + second_body.contains("viewer=second"), + "second response must not reuse the first visitor's body: {second_body}" + ); +} + #[test] #[ignore = "requires Docker and pre-built trusted-server-axum binary"] fn test_wordpress_axum() { diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 96389be78..d691d8583 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1046,29 +1046,30 @@ Disabled rules never match, and their matcher and policy validation is deferred until they are enabled. Rule IDs are always normalized and must remain nonempty and unique, including for disabled placeholders. -| Field | Type | Required | Description | -| -------------------------------- | ------------- | -------- | --------------------------------------------------------------- | -| `id` | String | Yes | Unique operator-facing rule identifier | -| `enabled` | Boolean | No | Whether the rule participates in matching (default `false`) | -| `preset` | String | Matcher | Built-in preset such as `nextjs-static` | -| `path_prefix` | String | Matcher | Request path prefix | -| `path_glob` | String | Matcher | Single glob matched against the request path | -| `path_globs` | Array[String] | Matcher | Multiple globs matched against the request path | -| `path_regex` | String | Matcher | Regex matched against the request path | -| `extensions` | Array[String] | Matcher | Case-insensitive file extensions | -| `fingerprint_style` | String | No | Required bundler fingerprint convention before matching | -| `visibility` | String | No | `public` or `private` (default `public`) | -| `browser_ttl_seconds` | Integer | Policy | Browser `max-age`; must be positive when `immutable = true` | -| `edge_ttl_seconds` | Integer | Policy | TTL emitted through the runtime-specific shared-cache directive | -| `stale_while_revalidate_seconds` | Integer | No | Optional `stale-while-revalidate` | -| `stale_if_error_seconds` | Integer | No | Optional `stale-if-error` | -| `immutable` | Boolean | No | Add `immutable` for a validated content-addressed rule | - -An enabled rule must configure exactly one matcher and at least one of -`browser_ttl_seconds` or `edge_ttl_seconds`. `path_glob` and `path_globs` are -mutually exclusive. `immutable = true` additionally requires a positive browser -TTL and either the content-addressed `nextjs-static` preset or an explicit -`fingerprint_style`. +| Field | Type | Required | Description | +| -------------------------------- | ------------- | -------- | ---------------------------------------------------------------------------------- | +| `id` | String | Yes | Unique operator-facing rule identifier | +| `enabled` | Boolean | No | Whether the rule participates in matching (default `false`) | +| `preset` | String | Matcher | Built-in preset such as `nextjs-static` | +| `path_prefix` | String | Matcher | Request path prefix | +| `path_glob` | String | Matcher | Single glob matched against the request path | +| `path_globs` | Array[String] | Matcher | Multiple globs matched against the request path | +| `path_regex` | String | Matcher | Regex matched against the request path | +| `extensions` | Array[String] | Matcher | Case-insensitive file extensions | +| `fingerprint_style` | String | No | Required bundler fingerprint convention before matching | +| `visibility` | String | No | `public` or `private` (default `public`) | +| `browser_ttl_seconds` | Integer | Policy | Browser `max-age`; required for private rules and positive with `immutable = true` | +| `edge_ttl_seconds` | Integer | Policy | Public rules only: TTL emitted through the runtime-specific shared-cache directive | +| `stale_while_revalidate_seconds` | Integer | No | Optional `stale-while-revalidate` | +| `stale_if_error_seconds` | Integer | No | Optional `stale-if-error` | +| `immutable` | Boolean | No | Add `immutable` for a validated content-addressed rule | + +An enabled rule must configure exactly one matcher. Public rules must configure +at least one of `browser_ttl_seconds` or `edge_ttl_seconds`; private rules must +configure `browser_ttl_seconds` and must not configure `edge_ttl_seconds`. +`path_glob` and `path_globs` are mutually exclusive. `immutable = true` +additionally requires a positive browser TTL and either the content-addressed +`nextjs-static` preset or an explicit `fingerprint_style`. The filename fingerprint check is intentionally conservative and style-specific. It examines the suffix immediately before the final extension and requires a @@ -1144,11 +1145,15 @@ built-in cache policy and do not require an asset rule. Shared-cache keys for `/static/tsjs=` must preserve `v`; otherwise a matching immutable response can collide with the missing or mismatched version's short-TTL response. -`edge_ttl_seconds` only emits the selected runtime's shared-cache directive. The -runtime or service must also enable and consume that directive. The checked-in -Cloudflare manifests enable Workers Cache. Fastly synthetic and final egress -responses still require explicit runtime cache integration, tracked in -[#908](https://github.com/IABTechLab/trusted-server/issues/908). +`edge_ttl_seconds` only emits the selected runtime's shared-cache directive for +public rules. The runtime or service must also enable and consume that +directive. The checked-in Cloudflare manifests intentionally do not enable +Workers Cache: the Worker serves the full publisher gateway, not an isolated +static-only entrypoint. Emitting `Cloudflare-CDN-Cache-Control` alone must not +be treated as permission to cache every response. Any future Workers Cache +opt-in must isolate or explicitly allowlist cacheable traffic. Fastly synthetic +and final egress responses still require explicit runtime cache integration, +tracked in [#908](https://github.com/IABTechLab/trusted-server/issues/908). ## Integration Configurations From 0e1a405f9c6e7f659cbdc6006a8ef37792719fa7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 09:35:38 +0530 Subject: [PATCH 292/395] Document admin diagnostics review fixes --- ...8-admin-diagnostics-review-fixes-design.md | 265 ++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md diff --git a/docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md b/docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md new file mode 100644 index 000000000..d19062725 --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md @@ -0,0 +1,265 @@ +# Admin diagnostics review fixes + +**PR:** #928 +**Date:** 2026-08-18 +**Status:** Approved design + +## Problem + +PR #928 adds authenticated operator diagnostics at `/_ts/admin/ec`, +`/_ts/admin/ec/{id}`, and `/_ts/admin/eids`. Review identified five issues: + +1. Startup authentication coverage checks the literal router template + `/_ts/admin/ec/{id}`, while runtime authentication checks concrete request + paths. A handler that matches only the literal braces can therefore pass + startup validation while leaving real EC IDs unauthenticated. +2. Non-GET requests and malformed or trailing diagnostic paths enter the + publisher fallback after successful authentication. That can forward the + admin `Authorization` header and request body to the publisher origin. +3. Fastly dispatches the EIDs diagnostic through normal EC setup and attaches + `EcFinalizeState`. Entry-point finalization can then ingest EID cookies and + write to KV even though the diagnostic is documented as read-only. +4. Parseable KV bodies and metadata are displayed by serializing typed schema + values. Unknown fields are dropped, and legacy representations such as + map-shaped `seen_domains` are normalized instead of being shown as stored. +5. The operator-facing API is missing from the API reference. + +## Goals + +- Require valid Basic authentication for every recognized admin request at + runtime, including concrete EC IDs. +- Reject invalid admin handler coverage during startup using a concrete EC ID + probe while preserving router-template diagnostics in error messages. +- Ensure diagnostic requests never enter publisher fallback, regardless of + supported method or malformed/trailing path shape. +- Keep `GET /_ts/admin/eids` read-only on Fastly by preventing all EC + finalization state from being attached. +- Display all parseable KV entry and metadata JSON without dropping or + normalizing stored fields. +- Preserve typed validation and auction derivation independently of the raw + display representation. +- Document authentication, requests, responses, status codes, cache policy, + and adapter limitations. +- Keep changes narrowly scoped to the new admin diagnostics. + +## Non-goals + +- Do not change authentication behavior for non-admin handler patterns. +- Do not change publisher fallback behavior outside the reserved admin + namespace. +- Do not add EC lookup support to Axum, Cloudflare, or Spin. +- Do not change live auction EID resolution or cookie-ingestion semantics. +- Do not add a new KV abstraction solely to spy on Fastly writes in tests. +- Do not redesign the admin API payload beyond preserving stored JSON and + documenting its existing derived fields. +- Do not push commits, reply to GitHub review threads, or resolve review + conversations as part of implementation. + +## Design + +### 1. Parameter-aware startup authentication coverage + +Keep the canonical admin route templates as the source used for coverage +errors and route-consistency tests. When testing whether a configured handler +covers `/_ts/admin/ec/{id}`, match the handler against a fixed representative +valid EC path instead of the literal template. The representative ID will use +fictional test data and satisfy the production `{64hex}.{6alnum}` format. + +All non-parameterized admin routes continue to use their canonical paths as +their coverage probes. A prefix handler such as `^/_ts/admin` therefore remains +valid, while a regex matching only literal braces is rejected at startup and +reported as failing to cover `/_ts/admin/ec/{id}`. + +### 2. Runtime authentication fails closed for admin paths + +`enforce_basic_auth` currently treats a missing matching handler as meaning the +request is public. Add a shared admin-namespace classifier with a segment +boundary: it recognizes `/_ts/admin` and paths beginning `/_ts/admin/`, but not +similar publisher paths such as `/_ts/administrator`. + +If no handler matches a recognized admin path, return a configuration error +instead of `Ok(None)`. Existing adapter middleware converts that error into a +local server response, so the request cannot reach a route handler or publisher +origin. If a handler matches, credential extraction and constant-time +comparison remain unchanged. + +This runtime invariant is defense in depth. Startup validation catches known +route misconfiguration, while runtime classification also protects malformed, +trailing, and future admin paths. + +### 3. Local denial before publisher fallback + +Add a shared core classifier for the EC/EIDs diagnostic path family and use it +at the top of each adapter's fallback dispatcher, before integration or +publisher handling. This avoids a repetitive route matrix and covers path +shapes that a fixed route table can miss. + +For the seven methods supported by publisher fallback (`GET`, `POST`, `HEAD`, +`OPTIONS`, `PUT`, `PATCH`, and `DELETE`), apply this contract after successful +authentication: + +- A non-GET request to `/_ts/admin/ec`, a single-segment + `/_ts/admin/ec/{id}`, or `/_ts/admin/eids` returns local `405 Method Not + Allowed` with `Allow: GET`. +- A path with a trailing slash, an extra segment, a missing segment structure, + or an EIDs suffix returns local `404 Not Found` and never reaches the + publisher. +- Existing GET handling remains unchanged: Fastly validates an explicit EC ID + and can return `400`, while portability adapters return their existing local + `501` for the two EC lookup forms. + +All denial responses use `Cache-Control: no-store`. Unsupported methods such as +`TRACE` continue to receive the router's local `405`; they are not registered +for publisher fallback and therefore cannot leak credentials upstream. + +The guard is duplicated only at the four adapter fallback entry points. Path +classification and response construction remain shared so status, headers, +and behavior cannot drift. + +### 4. Fastly EIDs dispatch skips EC setup and finalization + +Handle `NamedRouteHandler::AdminEidsLookup` in `execute_named` before request +filters and `build_ec_request_state`, next to the existing early batch-sync +branch. Build only the partner registry, call `handle_admin_eids_lookup`, map +errors through the existing HTTP error conversion, and return the response +without calling `attach_dispatch_extensions`. + +Basic authentication and standard response-header middleware remain outside +this dispatch function and continue to run. Because the returned response has +no `EcFinalizeState`, the Fastly entry point cannot call EC finalization, +`ingest_eid_cookies`, pull sync, or any EC KV write for this endpoint. + +The normal `run_named_route` EIDs arm will become unreachable or be removed in +the smallest form that keeps the enum dispatch exhaustive and clear. + +### 5. Separate raw display parsing from typed interpretation + +For entry bodies, perform two independent parses: + +1. Parse `lookup.body` as `serde_json::Value` for `payload.entry`. +2. Parse the same bytes as `KvEntry` only for tombstone calculation, schema + validation, and auction derivation. + +Add derived `created_iso` and `consent.updated_iso` fields directly to the raw +JSON object using its stored numeric timestamps. Never overwrite a stored field +with the same derived-field name. Unknown fields and legacy nested shapes stay +unchanged. + +The entry outcomes are: + +- Invalid JSON: omit `entry`; include `entry_error` and lossy UTF-8 `raw_body`; + omit tombstone and auction. +- Valid JSON but invalid `KvEntry`: include the raw `entry`; include + `entry_error`; omit tombstone and auction. +- Valid typed entry that fails validation: include raw `entry` and tombstone; + include the validation error; omit auction. +- Valid and validated typed entry: include raw `entry`, tombstone, and the + existing derived auction view. + +For metadata, parse bytes as `serde_json::Value` for display and independently +as `KvMetadata` for existing schema diagnostics. Parseable raw metadata remains +visible even if typed metadata parsing reports an error. Invalid JSON remains +omitted and its raw/error detail stays in `metadata_error`. + +Auction derivation continues to use `KvEntry`, `resolve_partner_ids`, and +`to_eids`, preserving production semantics. Live request consent remains a +documented limitation of the diagnostic view. + +### 6. Operator API documentation + +Add an Admin Diagnostic Endpoints section to +`docs/guide/api-reference.md` covering: + +- Basic authentication and the sensitivity of returned data. +- `GET /_ts/admin/ec` with ID resolution from the `ts-ec` cookie. +- `GET /_ts/admin/ec/{id}` and its explicit ID format. +- EC success fields, raw/typed error outcomes, auction derivation, `400`, + `404`, `405`, and `501` responses. +- Fastly-only EC lookup support and authenticated `501` responses from Axum, + Cloudflare, and Spin. +- `GET /_ts/admin/eids`, its cookie inputs, always-`200` diagnostic payload, + and support on every adapter. +- `Content-Type` and `Cache-Control: no-store` behavior. +- The three diagnostic paths in the protected-endpoint list. + +Examples use only reserved domains and fictional IDs. + +## Testing strategy + +Implementation follows red-green-refactor, one behavior at a time. + +### Core authentication and settings + +- A template-only handler configuration fails startup coverage for the + parameterized EC route. +- A concrete valid EC request under that configuration cannot be treated as + public by runtime auth. +- Existing broad and exact non-parameterized handler coverage remains valid. +- Similar non-admin prefixes remain public unless configured otherwise. + +### Cross-adapter routing + +For Fastly, Axum, Cloudflare, and Spin, authenticated requests verify: + +- Wrong supported methods on the bare EC, single-ID EC, and EIDs routes return + local `405` with `Allow: GET` and `no-store`. +- Trailing and extra-segment EC/EIDs paths return local `404` with `no-store`. +- Marker bodies and credentials do not reach publisher handling; deterministic + local status provides the existing adapter test seam for this invariant. +- Valid GET behavior remains `200`/domain error on Fastly, `501` for EC lookup + on portability adapters, and `200` for EIDs on every adapter. + +### Fastly read-only behavior + +An authenticated browser-shaped EIDs request carrying valid EC, EID, and +shared-ID cookies returns `200` without an `EcFinalizeState` response extension. +The Fastly entry point only performs EC writes when that extension is present, +so its absence proves the diagnostic cannot invoke the KV write path without +introducing test-only production seams. + +### Raw KV display + +- A parseable entry with unknown top-level and nested fields preserves those + exact values. +- Legacy map-shaped `seen_domains` remains a map with its nested history data. +- Parseable metadata preserves unknown fields. +- Stored timestamps remain unchanged and ISO companions are added without + overwriting stored collisions. +- Typed parsing still produces the expected auction view from the same entry. +- Valid JSON with an invalid typed schema remains visible with `entry_error`. + +### Verification + +After targeted tests pass, run the repository-required checks relevant to all +touched crates and documentation: + +- `cargo fmt --all -- --check` +- `cargo test-fastly` +- `cargo test-axum` +- `cargo test-cloudflare` +- `cargo test-spin` +- `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity` +- `cargo clippy-fastly` +- `cargo clippy-axum` +- `cargo clippy-cloudflare` +- `cargo clippy-cloudflare-wasm` +- `cargo clippy-spin-native` +- `cargo clippy-spin-wasm` +- `cd docs && npm run format` + +JS sources are untouched; JS build, test, and format gates are not required for +the implementation-specific verification unless another change introduces a +JS dependency. + +## Risks and mitigations + +- **Overbroad runtime auth classification:** use an exact namespace segment + boundary and add a similar-prefix regression. +- **Adapter behavior drift:** share path classification and denial response + construction; keep only the fallback entry-point call adapter-local. +- **Route precedence regressions:** preserve named GET handlers and guard only + requests that reached fallback. +- **Accidental raw-data normalization:** assert unknown fields and legacy + structures on the final serialized handler response, not only helper values. +- **Fastly write regression:** assert the structural finalization gate is absent + from the response. From ec8ca8c04ef27256846e96792afe84606968e5e0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 09:37:30 +0530 Subject: [PATCH 293/395] Clarify admin authentication probes --- ...8-admin-diagnostics-review-fixes-design.md | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md b/docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md index d19062725..f6cf8aa06 100644 --- a/docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md +++ b/docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md @@ -60,16 +60,24 @@ PR #928 adds authenticated operator diagnostics at `/_ts/admin/ec`, ### 1. Parameter-aware startup authentication coverage Keep the canonical admin route templates as the source used for coverage -errors and route-consistency tests. When testing whether a configured handler -covers `/_ts/admin/ec/{id}`, match the handler against a fixed representative -valid EC path instead of the literal template. The representative ID will use -fictional test data and satisfy the production `{64hex}.{6alnum}` format. +errors and route-consistency tests. Define one canonical mapping from each +template to its concrete authentication probe. When testing whether a +configured handler covers `/_ts/admin/ec/{id}`, match the handler against a +fixed representative valid EC path instead of the literal template. The +representative ID will use fictional test data and satisfy the production +`{64hex}.{6alnum}` format. All non-parameterized admin routes continue to use their canonical paths as their coverage probes. A prefix handler such as `^/_ts/admin` therefore remains valid, while a regex matching only literal braces is rejected at startup and reported as failing to cover `/_ts/admin/ec/{id}`. +Use the same template-to-probe mapping everywhere settings validation decides +whether a handler protects an admin endpoint. This includes both uncovered +endpoint detection and placeholder-password rejection. A handler that protects +concrete EC IDs must therefore be recognized as an admin handler for credential +strength validation even when it does not match the literal router template. + ### 2. Runtime authentication fails closed for admin paths `enforce_basic_auth` currently treats a missing matching handler as meaning the @@ -174,7 +182,7 @@ Add an Admin Diagnostic Endpoints section to - `GET /_ts/admin/ec` with ID resolution from the `ts-ec` cookie. - `GET /_ts/admin/ec/{id}` and its explicit ID format. - EC success fields, raw/typed error outcomes, auction derivation, `400`, - `404`, `405`, and `501` responses. + `401`, `404`, `405`, and `501` responses. - Fastly-only EC lookup support and authenticated `501` responses from Axum, Cloudflare, and Spin. - `GET /_ts/admin/eids`, its cookie inputs, always-`200` diagnostic payload, @@ -192,6 +200,8 @@ Implementation follows red-green-refactor, one behavior at a time. - A template-only handler configuration fails startup coverage for the parameterized EC route. +- A concrete-ID handler with a placeholder password is still rejected as an + admin handler through the shared template-to-probe mapping. - A concrete valid EC request under that configuration cannot be treated as public by runtime auth. - Existing broad and exact non-parameterized handler coverage remains valid. From 2ec38e74284cb436c45b39bf64aac747c9bb8631 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 10:09:17 +0530 Subject: [PATCH 294/395] Plan admin diagnostics review fixes --- ...26-08-18-admin-diagnostics-review-fixes.md | 576 ++++++++++++++++++ 1 file changed, 576 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-18-admin-diagnostics-review-fixes.md diff --git a/docs/superpowers/plans/2026-08-18-admin-diagnostics-review-fixes.md b/docs/superpowers/plans/2026-08-18-admin-diagnostics-review-fixes.md new file mode 100644 index 000000000..b13430a45 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-admin-diagnostics-review-fixes.md @@ -0,0 +1,576 @@ +# Admin Diagnostics Review Fixes Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close every PR #928 review finding by making admin diagnostics fail closed, preventing publisher/KV side effects, preserving raw KV JSON, and documenting the API. + +**Architecture:** Core settings owns the canonical admin-template-to-auth-probe mapping and runtime admin namespace classification. Core EC admin code owns one shared fallback-denial response so each adapter only adds a small guard at its publisher fallback boundary. Fastly dispatches the read-only EIDs diagnostic before EC setup, while raw JSON display and typed interpretation remain separate inside the core handler. + +**Tech Stack:** Rust 2024, `http`, `serde_json`, `error-stack`, EdgeZero adapter routers, Fastly/Viceroy, Markdown documentation. + +**Design spec:** `docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md` + +--- + +## File map + +- Modify `crates/trusted-server-core/src/settings.rs`: canonical admin route/auth + probes, admin namespace classification, startup validation, and tests. +- Modify `crates/trusted-server-core/src/auth.rs`: runtime fail-closed behavior + and regression tests. +- Modify `crates/trusted-server-core/src/ec/admin.rs`: shared diagnostic + fallback denial, lossless raw JSON display, and unit tests. +- Modify `crates/trusted-server-adapter-fastly/src/app.rs`: fallback guard, + early read-only EIDs dispatch, and adapter tests. +- Modify each portability adapter's `src/app.rs` and `tests/routes.rs`: fallback + guard and cross-adapter route regressions. +- Modify `docs/guide/api-reference.md`: operator-facing contract. + +No new crate, dependency, schema type, or test-only production seam is needed. + +### Task 1: Make admin authentication coverage parameter-aware and fail closed + +**Files:** + +- Modify: `crates/trusted-server-core/src/settings.rs:2194-2279` +- Modify: `crates/trusted-server-core/src/settings.rs:4876-4970` +- Modify: `crates/trusted-server-core/src/auth.rs:29-55` +- Test: `crates/trusted-server-core/src/auth.rs:79-315` + +- [ ] **Step 1: Add a failing literal-template startup regression** + +Build settings TOML whose first handler covers the four non-parameterized admin +routes and whose second handler covers only literal braces: + +```rust +[[handlers]] +path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" +username = "admin" +password = "strong-test-password" + +[[handlers]] +path = "^/_ts/admin/ec/[{]id[}]$" +username = "admin" +password = "strong-test-password" +``` + +Assert `Settings::from_toml` fails and identifies `/_ts/admin/ec/{id}` as +uncovered. + +- [ ] **Step 2: Run the regression and verify RED** + +```bash +cargo test-fastly from_toml_rejects_literal_parameter_template_auth_coverage +``` + +Expected: FAIL because current validation accepts the literal template match. + +- [ ] **Step 3: Add and run a failing concrete-handler password regression** + +Add settings with a concrete-ID handler regex +`^/_ts/admin/ec/[a-f0-9]{64}[.][a-z0-9]{6}$` and placeholder password +`change-me-admin-password`. Assert finalization rejects it as an admin handler. + +```bash +cargo test-fastly from_toml_rejects_placeholder_password_for_concrete_admin_ec_handler +``` + +Expected: FAIL because password validation also uses the literal template. + +- [ ] **Step 4: Implement one canonical template-to-auth-probe mapping** + +Keep `Settings::ADMIN_ENDPOINTS` as canonical templates. Add a fixed fictional +valid EC probe and a helper used by both coverage and password validation: + +```rust +const ADMIN_EC_ID_AUTH_PROBE: &str = concat!( + "/_ts/admin/ec/", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ".abc123", +); + +fn admin_auth_probe(path: &'static str) -> &'static str { + match path { + "/_ts/admin/ec/{id}" => ADMIN_EC_ID_AUTH_PROBE, + path => path, + } +} +``` + +Make `uncovered_admin_endpoints` report the template but match its probe. Make +`validate_admin_handler_passwords` use the same helper. Update stale comments. + +- [ ] **Step 5: Run both settings regressions and verify GREEN** + +```bash +cargo test-fastly literal_parameter_template_auth_coverage +cargo test-fastly placeholder_password_for_concrete_admin_ec_handler +``` + +Expected: PASS. + +- [ ] **Step 6: Add failing runtime fail-closed auth tests** + +In `auth.rs`, deserialize settings directly with `toml::from_str` to bypass +startup finalization. With the literal-template-only configuration, send a +concrete valid EC request and assert `enforce_basic_auth` returns a configuration +error rather than `Ok(None)`. Add `/_ts/administrator` as a boundary case that +must remain public when no handler matches. + +- [ ] **Step 7: Run runtime tests and verify RED** + +```bash +cargo test-fastly concrete_admin_path_without_matching_handler_fails_closed +``` + +Expected: FAIL because current auth returns `Ok(None)`. + +- [ ] **Step 8: Implement the runtime namespace invariant** + +Add: + +```rust +#[must_use] +pub fn is_admin_path(path: &str) -> bool { + path == "/_ts/admin" || path.starts_with("/_ts/admin/") +} +``` + +When `handler_for_path` returns `None`, make `enforce_basic_auth` return +`TrustedServerError::Configuration` for an admin path and retain `Ok(None)` for +all other paths. + +- [ ] **Step 9: Run core tests and target-matched suite** + +```bash +cargo test-fastly admin_path +cargo test-fastly uncovered_admin_endpoints +cargo test-fastly +``` + +Expected: PASS without warnings. + +- [ ] **Step 10: Commit Task 1** + +```bash +git add crates/trusted-server-core/src/settings.rs crates/trusted-server-core/src/auth.rs +git commit -m "Fail closed for concrete admin routes" +``` + +### Task 2: Add a shared local denial for diagnostic fallback requests + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/admin.rs:20-45` +- Modify: `crates/trusted-server-core/src/ec/admin.rs:432-464` +- Test: `crates/trusted-server-core/src/ec/admin.rs:464-925` + +- [ ] **Step 1: Add failing table-driven fallback tests** + +Test a new +`deny_admin_diagnostic_fallback(&Request) -> Option>` +helper. For bare EC, single-ID EC, and EIDs, every non-GET publisher fallback +method must return `405`, `Allow: GET`, and `Cache-Control: no-store`. GET and +non-GET requests to trailing/extra-segment EC/EIDs forms must return `404` and +`no-store`. An unrelated publisher path must return `None`. + +- [ ] **Step 2: Run focused tests and verify RED** + +```bash +cargo test-fastly admin_diagnostic_fallback +``` + +Expected: compilation FAIL because the helper does not exist. + +- [ ] **Step 3: Implement classification and response construction** + +Add a private path-shape classifier and documented public helper. Its core +response logic is: + +```rust +let mut response = if shape.is_valid_resource() && req.method() != Method::GET { + json_error(StatusCode::METHOD_NOT_ALLOWED, "method not allowed") +} else { + json_error(StatusCode::NOT_FOUND, "admin diagnostic route not found") +}; +if response.status() == StatusCode::METHOD_NOT_ALLOWED { + response + .headers_mut() + .insert(header::ALLOW, HeaderValue::from_static("GET")); +} +``` + +Reuse `json_error`/`json_response`, which already set JSON and `no-store`. +Classify only the EC/EIDs families. EC bare and exactly one non-empty ID segment +are valid resource shapes; EIDs exact is valid; suffix/trailing forms are +malformed. A valid GET that somehow reaches fallback returns local `404`. + +- [ ] **Step 4: Run focused and target-matched tests** + +```bash +cargo test-fastly admin_diagnostic_fallback +cargo test-fastly +``` + +Expected: PASS. + +- [ ] **Step 5: Commit Task 2** + +```bash +git add crates/trusted-server-core/src/ec/admin.rs +git commit -m "Deny admin diagnostics in publisher fallback" +``` + +### Task 3: Wire the denial guard into every adapter + +**Files:** + +- Modify/Test: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-axum/src/app.rs` +- Test: `crates/trusted-server-adapter-axum/tests/routes.rs` +- Modify: `crates/trusted-server-adapter-cloudflare/src/app.rs` +- Test: `crates/trusted-server-adapter-cloudflare/tests/routes.rs` +- Modify: `crates/trusted-server-adapter-spin/src/app.rs` +- Test: `crates/trusted-server-adapter-spin/tests/routes.rs` + +- [ ] **Step 1: Add Fastly adapter regressions and verify RED** + +Using authenticated router requests, test the valid diagnostic shapes against +`POST`, `HEAD`, `OPTIONS`, `PUT`, `PATCH`, and `DELETE`, asserting `405`, +`Allow: GET`, and `no-store`. Test authenticated GET and POST requests for +trailing/extra-segment forms, asserting `404` and `no-store`. + +```bash +cargo test-fastly authenticated_admin_diagnostic_fallback +``` + +Expected: FAIL because requests enter publisher fallback. + +- [ ] **Step 2: Wire Fastly and verify GREEN** + +Import the helper and call it at the start of `dispatch_fallback`, before GPT +preparation, filters, integration routing, or publisher handling: + +```rust +if let Some(response) = deny_admin_diagnostic_fallback(&req) { + return response; +} +``` + +```bash +cargo test-fastly authenticated_admin_diagnostic_fallback +cargo test-fastly +``` + +Expected: PASS. + +- [ ] **Step 3: Add Axum regressions and verify RED** + +Add the same matrices in `tests/routes.rs` using `make_service()`. + +```bash +cargo test-axum authenticated_admin_diagnostic_fallback +``` + +Expected: FAIL. + +- [ ] **Step 4: Wire Axum and verify GREEN** + +Call the helper at the start of Axum's fallback `dispatch` before publisher +handling. + +```bash +cargo test-axum authenticated_admin_diagnostic_fallback +cargo test-axum +``` + +Expected: PASS. + +- [ ] **Step 5: Add Cloudflare regressions and verify RED** + +Use `request_builder()` plus `route(test_router(), req)`. + +```bash +cargo test-cloudflare authenticated_admin_diagnostic_fallback +``` + +Expected: FAIL. + +- [ ] **Step 6: Wire Cloudflare and verify GREEN** + +Call the helper before Cloudflare integration/publisher dispatch. + +```bash +cargo test-cloudflare authenticated_admin_diagnostic_fallback +cargo test-cloudflare +``` + +Expected: PASS. + +- [ ] **Step 7: Add Spin regressions and verify RED** + +Use the existing Spin router helpers with the same matrices. + +```bash +cargo test-spin authenticated_admin_diagnostic_fallback +``` + +Expected: FAIL. + +- [ ] **Step 8: Wire Spin and verify GREEN** + +Call the helper before Spin integration/publisher dispatch. + +```bash +cargo test-spin authenticated_admin_diagnostic_fallback +cargo test-spin +``` + +Expected: PASS. + +- [ ] **Step 9: Commit Task 3** + +```bash +git add crates/trusted-server-adapter-fastly/src/app.rs \ + crates/trusted-server-adapter-axum/src/app.rs \ + crates/trusted-server-adapter-axum/tests/routes.rs \ + crates/trusted-server-adapter-cloudflare/src/app.rs \ + crates/trusted-server-adapter-cloudflare/tests/routes.rs \ + crates/trusted-server-adapter-spin/src/app.rs \ + crates/trusted-server-adapter-spin/tests/routes.rs +git commit -m "Keep admin diagnostics out of publisher fallback" +``` + +### Task 4: Make Fastly EIDs diagnostics structurally read-only + +**Files:** + +- Modify/Test: `crates/trusted-server-adapter-fastly/src/app.rs:501-595` +- Reference: `crates/trusted-server-adapter-fastly/src/main.rs:184-247` +- Reference: `crates/trusted-server-core/src/ec/finalize.rs:77-106` + +- [ ] **Step 1: Add and run a failing finalization-state regression** + +Create an authenticated browser-shaped `GET /_ts/admin/eids` request with valid +EC, EID, and shared-ID cookies. Assert `200` and: + +```rust +assert!( + response.extensions().get::().is_none(), + "admin EIDs diagnostics should not attach EC finalization state" +); +``` + +```bash +cargo test-fastly admin_eids_diagnostic_skips_ec_finalization +``` + +Expected: FAIL because `execute_named` attaches `EcFinalizeState`. + +- [ ] **Step 2: Add the early EIDs dispatch** + +Before GPT preparation or EC setup, build the registry, call the handler, map +errors through `http_error`, and return without `attach_dispatch_extensions`: + +```rust +if matches!(handler, NamedRouteHandler::AdminEidsLookup) { + let result = PartnerRegistry::from_config(&state.settings.ec.partners) + .and_then(|registry| handle_admin_eids_lookup(®istry, &req)); + return Ok(result.unwrap_or_else(|error| http_error(&error))); +} +``` + +Make the normal route arm explicitly unreachable or remove it cleanly. Update +module lifecycle comments. + +- [ ] **Step 3: Run focused and Fastly suites** + +```bash +cargo test-fastly admin_eids_diagnostic_skips_ec_finalization +cargo test-fastly +``` + +Expected: PASS. + +- [ ] **Step 4: Commit Task 4** + +```bash +git add crates/trusted-server-adapter-fastly/src/app.rs +git commit -m "Keep admin EID diagnostics read only" +``` + +### Task 5: Preserve raw parseable KV entries and metadata + +**Files:** + +- Modify/Test: `crates/trusted-server-core/src/ec/admin.rs:47-278` +- Modify/Test: `crates/trusted-server-core/src/ec/admin.rs:464-925` + +- [ ] **Step 1: Add and run a failing lossless-display regression** + +Seed a valid raw entry with unknown top-level, consent, and partner fields; +legacy map-shaped `seen_domains`; valid auction partner data; and metadata with +an unknown field. Assert the final response preserves all raw values and shape, +keeps numeric timestamps, adds ISO companions, preserves metadata, and still +derives auction EIDs. + +```bash +cargo test-fastly parseable_legacy_entry_and_metadata_preserve_raw_json +``` + +Expected: FAIL because typed reserialization drops and normalizes data. + +- [ ] **Step 2: Add and run failing schema/collision tests** + +For valid JSON that cannot deserialize as `KvEntry`, assert raw `entry` remains +present with `entry_error` and no `raw_body`. For stored `created_iso` and +`consent.updated_iso`, assert neither is overwritten. + +```bash +cargo test-fastly valid_json_with_invalid_kv_schema_remains_visible +cargo test-fastly stored_iso_fields_are_not_overwritten +``` + +Expected: FAIL. + +- [ ] **Step 3: Separate raw display from typed interpretation** + +Parse `lookup.body` as `JsonValue` for `payload.entry`, then independently as +`KvEntry` for tombstone, validation, and auction. Invalid JSON sets +`entry_error` plus lossy `raw_body`; valid JSON with an invalid schema remains +visible and sets only `entry_error`. + +Replace `entry_json_with_iso_timestamps(&KvEntry)` with a helper accepting +`&mut JsonValue`. Read raw `created` and `consent.updated` as `u64`, and use +`entry(...).or_insert(...)` for ISO companions so stored collisions win. + +Parse metadata independently as `JsonValue` for display and `KvMetadata` only +for diagnostics. Typed metadata failure must not erase parseable raw metadata; +invalid JSON remains in `metadata_error`. + +- [ ] **Step 4: Run admin and Fastly suites** + +```bash +cargo test-fastly ec::admin::tests +cargo test-fastly +``` + +Expected: PASS, including existing corrupt-entry, validation, timestamp, and +auction tests. + +- [ ] **Step 5: Commit Task 5** + +```bash +git add crates/trusted-server-core/src/ec/admin.rs +git commit -m "Preserve raw admin EC diagnostic records" +``` + +### Task 6: Document the operator API contract + +**Files:** + +- Modify: `docs/guide/api-reference.md:1-20` +- Modify: `docs/guide/api-reference.md:497-590` +- Modify: `docs/guide/api-reference.md:746-772` + +- [ ] **Step 1: Add the Admin Diagnostic Endpoints section** + +Document Basic Auth and sensitive data; both EC lookup forms; response fields +and the raw/typed error matrix; `401`, `400`, `404`, `405`, and `501`; Fastly +support and portability `501`; EIDs cookie inputs, payload, always-`200` +post-auth semantics, and all-adapter support; JSON/no-store behavior; `Allow: +GET`; and the live-consent limitation. Use only fictional/example data. + +- [ ] **Step 2: Update navigation and protected endpoints** + +Add the section to the API category list and the three routes to Protected +Endpoints. + +- [ ] **Step 3: Format and inspect documentation** + +```bash +cd docs && npm run format +git diff --check +git diff -- docs/guide/api-reference.md +``` + +Expected: formatting passes, no whitespace errors, and the contract matches +implemented statuses and headers. + +- [ ] **Step 4: Commit Task 6** + +```bash +git add docs/guide/api-reference.md +git commit -m "Document admin EC and EID diagnostics" +``` + +### Task 7: Verify the complete review resolution + +**Files:** Verify all files changed by Tasks 1-6. + +- [ ] **Step 1: Run formatting** + +```bash +cargo fmt --all -- --check +cd docs && npm run format +``` + +Expected: PASS. + +- [ ] **Step 2: Run all adapter tests** + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: PASS. + +- [ ] **Step 3: Run parity tests** + +```bash +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +``` + +Expected: PASS. + +- [ ] **Step 4: Run all target-matched lint gates** + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: PASS with `-D warnings`. + +- [ ] **Step 5: Inspect final branch state** + +```bash +git diff main...HEAD --check +git status --short +git log --oneline --decorate -10 +``` + +Expected: no uncommitted implementation changes and a focused commit sequence. + +- [ ] **Step 6: Request code review** + +Invoke `superpowers:requesting-code-review` with the approved spec and plan. +Address only verified findings and rerun affected tests after corrections. + +- [ ] **Step 7: Verify before completion** + +Invoke `superpowers:verification-before-completion`, confirm fresh output for +every claimed gate, and report environmental limitations rather than claiming +success. + +- [ ] **Step 8: Prepare review-thread resolution notes** + +Map each of the five findings to its implementing commit and test evidence. Do +not post or resolve GitHub threads without separate user authorization. From e637524aab482679c835f1f8a3a5fe32206cf9dc Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 10:22:00 +0530 Subject: [PATCH 295/395] Fail closed for concrete admin routes --- crates/trusted-server-core/src/auth.rs | 61 +++++++++++++++- crates/trusted-server-core/src/settings.rs | 84 ++++++++++++++++++++-- 2 files changed, 137 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-core/src/auth.rs b/crates/trusted-server-core/src/auth.rs index 8e70aa020..ecc2fdb8f 100644 --- a/crates/trusted-server-core/src/auth.rs +++ b/crates/trusted-server-core/src/auth.rs @@ -20,7 +20,9 @@ const BASIC_AUTH_REALM: &str = r#"Basic realm="Trusted Server""#; /// Admin endpoints are protected by requiring a handler during settings /// finalization; see [`Settings::from_toml`]. Credential checks use constant-time /// comparison for both username and password, and evaluate both regardless of -/// individual match results to avoid timing oracles. +/// individual match results to avoid timing oracles. Runtime requests within +/// the reserved admin namespace fail closed if no handler matches, providing +/// defense in depth for malformed and parameterized paths. /// /// # Errors /// @@ -30,7 +32,13 @@ pub fn enforce_basic_auth( settings: &Settings, req: &Request, ) -> Result>, Report> { - let Some(handler) = settings.handler_for_path(req.uri().path())? else { + let path = req.uri().path(); + let Some(handler) = settings.handler_for_path(path)? else { + if Settings::is_admin_path(path) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("Admin path `{path}` has no configured handler"), + })); + } return Ok(None); }; @@ -304,4 +312,53 @@ mod tests { .expect("should challenge admin path with missing credentials"); assert_eq!(response.status(), StatusCode::UNAUTHORIZED); } + + #[test] + fn concrete_admin_path_without_matching_handler_fails_closed() { + let config = crate_test_settings_str().replace( + r#"path = "^/_ts/admin" + username = "admin" + password = "admin-pass""#, + r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" + username = "admin" + password = "strong-test-password" + + [[handlers]] + path = "^/_ts/admin/ec/[{]id[}]$" + username = "admin" + password = "strong-test-password""#, + ); + let settings: Settings = + toml::from_str(&config).expect("should deserialize settings without finalization"); + let ec_id = format!("{}.abc123", "a".repeat(64)); + let req = build_request( + Method::GET, + &format!("https://example.com/_ts/admin/ec/{ec_id}"), + ); + + let error = enforce_basic_auth(&settings, &req) + .expect_err("should fail closed without a matching admin handler"); + assert!( + error.to_string().contains("no configured handler"), + "should describe the missing admin handler" + ); + } + + #[test] + fn similar_non_admin_prefix_without_handler_remains_public() { + let config = crate_test_settings_str().replace( + r#"path = "^/_ts/admin""#, + r#"path = "^/_ts/admin/keys/rotate$""#, + ); + let settings: Settings = + toml::from_str(&config).expect("should deserialize settings without finalization"); + let req = build_request(Method::GET, "https://example.com/_ts/administrator"); + + assert!( + enforce_basic_auth(&settings, &req) + .expect("should evaluate auth") + .is_none(), + "should not classify a similar prefix as the admin namespace" + ); + } } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index edb656489..8ba010901 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2191,6 +2191,13 @@ impl Settings { Ok(None) } + /// Returns whether `path` is within the reserved Trusted Server admin + /// namespace. + #[must_use] + pub(crate) fn is_admin_path(path: &str) -> bool { + path == "/_ts/admin" || path.starts_with("/_ts/admin/") + } + /// Known admin endpoint paths that must be covered by a handler. /// /// [`from_toml`](Self::from_toml) rejects configurations @@ -2199,10 +2206,10 @@ impl Settings { /// Update [`ADMIN_ENDPOINTS`](Self::ADMIN_ENDPOINTS) when adding new /// admin routes to `crates/trusted-server-adapter-fastly/src/app.rs`. /// - /// The `/_ts/admin/ec/{id}` entry is the literal router pattern; handler - /// path regexes are matched against it verbatim, so prefix-style admin - /// regexes (e.g. `^/_ts/admin`) cover it while regexes too narrow to - /// cover the parameterized route are rejected fail-closed. + /// The `/_ts/admin/ec/{id}` entry is the canonical router pattern. Handler + /// coverage is checked against a representative concrete EC ID via + /// [`admin_auth_probe`](Self::admin_auth_probe), while validation errors + /// continue to report this operator-facing route template. pub(crate) const ADMIN_ENDPOINTS: &[&str] = &[ "/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate", @@ -2211,6 +2218,19 @@ impl Settings { "/_ts/admin/eids", ]; + const ADMIN_EC_ID_AUTH_PROBE: &str = concat!( + "/_ts/admin/ec/", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ".abc123", + ); + + fn admin_auth_probe(path: &'static str) -> &'static str { + match path { + "/_ts/admin/ec/{id}" => Self::ADMIN_EC_ID_AUTH_PROBE, + path => path, + } + } + /// Returns admin endpoint paths that no configured handler covers. /// /// Called during settings finalization to enforce that every admin endpoint @@ -2227,7 +2247,7 @@ impl Settings { for &path in Self::ADMIN_ENDPOINTS { let mut covered = false; for h in &self.handlers { - if h.matches_path(path)? { + if h.matches_path(Self::admin_auth_probe(path))? { covered = true; break; } @@ -2265,7 +2285,9 @@ impl Settings { let covers_admin = Self::ADMIN_ENDPOINTS .iter() .try_fold(false, |covered, path| { - handler.matches_path(path).map(|matches| covered || matches) + handler + .matches_path(Self::admin_auth_probe(path)) + .map(|matches| covered || matches) })?; if covers_admin && is_admin_placeholder_password(handler.password.expose()) { @@ -4934,6 +4956,56 @@ origin_host_header_overide = "www.example.com""#, ); } + #[test] + fn from_toml_rejects_literal_parameter_template_auth_coverage() { + let toml_str = crate_test_settings_str().replace( + r#"path = "^/_ts/admin" + username = "admin" + password = "admin-pass""#, + r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" + username = "admin" + password = "strong-test-password" + + [[handlers]] + path = "^/_ts/admin/ec/[{]id[}]$" + username = "admin" + password = "strong-test-password""#, + ); + + let error = Settings::from_toml(&toml_str) + .expect_err("should reject literal parameter-template auth coverage"); + let message = format!("{error:?}"); + assert!( + message.contains("/_ts/admin/ec/{id}"), + "should identify the concrete EC route as uncovered, got: {message}" + ); + } + + #[test] + fn from_toml_rejects_placeholder_password_for_concrete_admin_ec_handler() { + let toml_str = crate_test_settings_str().replace( + r#"path = "^/_ts/admin" + username = "admin" + password = "admin-pass""#, + r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" + username = "admin" + password = "strong-test-password" + + [[handlers]] + path = "^/_ts/admin/ec/[a-f0-9]{64}[.][a-z0-9]{6}$" + username = "admin" + password = "change-me-admin-password""#, + ); + + let error = Settings::from_toml(&toml_str) + .expect_err("should reject placeholder password on concrete EC handler"); + let message = format!("{error:?}"); + assert!( + message.contains("placeholder password"), + "should identify the placeholder admin password, got: {message}" + ); + } + #[test] fn from_toml_and_env_rejects_config_without_admin_handler() { let origin_key = format!( From 539bebaa181b17de6bf95939a8a04f115af42b8c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 10:24:10 +0530 Subject: [PATCH 296/395] Deny admin diagnostics in publisher fallback --- crates/trusted-server-core/src/ec/admin.rs | 142 ++++++++++++++++++++- 1 file changed, 141 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index a4e6465ab..0e081275b 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -16,7 +16,7 @@ //! auth-gated and operator-facing, responses intentionally include full //! internal detail (raw consent strings, partner UIDs, parse errors). -use http::{Request, Response, StatusCode, header}; +use http::{HeaderValue, Method, Request, Response, StatusCode, header}; use serde::Serialize; use serde_json::Value as JsonValue; @@ -42,6 +42,64 @@ use super::registry::PartnerRegistry; /// Route prefix shared by the cookie-based and explicit-ID lookup routes. const ADMIN_EC_PATH: &str = "/_ts/admin/ec"; +/// Route used by the request-only EID cookie diagnostic. +const ADMIN_EIDS_PATH: &str = "/_ts/admin/eids"; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum AdminDiagnosticShape { + ValidResource, + Malformed, +} + +fn admin_diagnostic_shape(path: &str) -> Option { + if path == ADMIN_EC_PATH || path == ADMIN_EIDS_PATH { + return Some(AdminDiagnosticShape::ValidResource); + } + + if let Some(remainder) = path.strip_prefix("/_ts/admin/ec/") { + return Some(if !remainder.is_empty() && !remainder.contains('/') { + AdminDiagnosticShape::ValidResource + } else { + AdminDiagnosticShape::Malformed + }); + } + + path.starts_with("/_ts/admin/eids/") + .then_some(AdminDiagnosticShape::Malformed) +} + +/// Returns a local denial response when an admin diagnostic request reaches +/// an adapter's publisher fallback. +/// +/// Valid diagnostic resources reject non-GET methods with `405 Method Not +/// Allowed`. Malformed, trailing, and any valid GET route that unexpectedly +/// reaches fallback return `404 Not Found`. Unrelated publisher paths return +/// `None` so normal fallback behavior remains unchanged. +#[must_use] +pub fn deny_admin_diagnostic_fallback( + req: &Request, +) -> Option> { + let shape = admin_diagnostic_shape(req.uri().path())?; + let mut response = if shape == AdminDiagnosticShape::ValidResource + && req.method() != Method::GET + { + json_error(StatusCode::METHOD_NOT_ALLOWED, "method not allowed") + } else { + json_error( + StatusCode::NOT_FOUND, + "admin diagnostic route not found", + ) + }; + + if response.status() == StatusCode::METHOD_NOT_ALLOWED { + response + .headers_mut() + .insert(header::ALLOW, HeaderValue::from_static("GET")); + } + + Some(response) +} + /// Successful admin EC lookup payload. #[derive(Debug, Serialize)] struct AdminEcLookupResponse { @@ -520,6 +578,14 @@ mod tests { .expect("should build test request") } + fn request_with_method(method: http::Method, path: &str) -> Request { + Request::builder() + .method(method) + .uri(format!("https://edge.example.com{path}")) + .body(EdgeBody::empty()) + .expect("should build test request") + } + fn kv_with_entry(ec_id: &str, entry: &KvEntry) -> KvIdentityGraph { let kv = KvIdentityGraph::in_memory("test-store"); kv.create(ec_id, entry).expect("should seed KV entry"); @@ -565,6 +631,80 @@ mod tests { entry } + #[test] + fn admin_diagnostic_fallback_rejects_wrong_methods_locally() { + let ec_id = test_ec_id(); + let paths = [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{ec_id}"), + "/_ts/admin/eids".to_owned(), + ]; + let methods = [ + http::Method::POST, + http::Method::HEAD, + http::Method::OPTIONS, + http::Method::PUT, + http::Method::PATCH, + http::Method::DELETE, + ]; + + for path in paths { + for method in &methods { + let request = request_with_method(method.clone(), &path); + let response = deny_admin_diagnostic_fallback(&request) + .unwrap_or_else(|| panic!("should deny {method} {path} locally")); + + assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED); + assert_eq!( + response.headers().get(header::ALLOW), + Some(&http::HeaderValue::from_static("GET")), + "should advertise GET for {path}" + ); + assert_eq!( + response.headers().get(header::CACHE_CONTROL), + Some(&http::HeaderValue::from_static("no-store")), + "should prevent caching for {path}" + ); + } + } + } + + #[test] + fn admin_diagnostic_fallback_rejects_malformed_paths_locally() { + let ec_id = test_ec_id(); + let paths = [ + "/_ts/admin/ec/".to_owned(), + format!("/_ts/admin/ec/{ec_id}/extra"), + "/_ts/admin/eids/".to_owned(), + "/_ts/admin/eids/extra".to_owned(), + ]; + + for path in paths { + for method in [http::Method::GET, http::Method::POST] { + let request = request_with_method(method.clone(), &path); + let response = deny_admin_diagnostic_fallback(&request) + .unwrap_or_else(|| panic!("should deny {method} {path} locally")); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + response.headers().get(header::CACHE_CONTROL), + Some(&http::HeaderValue::from_static("no-store")), + "should prevent caching for {path}" + ); + } + } + } + + #[test] + fn admin_diagnostic_fallback_ignores_unrelated_publisher_paths() { + let request = request_with_method(http::Method::POST, "/articles/example"); + + assert!( + deny_admin_diagnostic_fallback(&request).is_none(), + "should leave unrelated publisher fallback unchanged" + ); + } + #[test] fn returns_entry_with_auction_view() { let ec_id = test_ec_id(); From 17f0ac40e04763f3085ec955556a93b3ac595ca7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 10:29:18 +0530 Subject: [PATCH 297/395] Keep admin diagnostics out of publisher fallback --- crates/trusted-server-adapter-axum/src/app.rs | 6 +- .../tests/routes.rs | 76 ++++++++++++++++++ .../src/app.rs | 5 +- .../tests/routes.rs | 64 +++++++++++++++ .../trusted-server-adapter-fastly/src/app.rs | 80 ++++++++++++++++++- crates/trusted-server-adapter-spin/src/app.rs | 5 +- .../tests/routes.rs | 64 +++++++++++++++ crates/trusted-server-core/src/ec/admin.rs | 20 ++--- 8 files changed, 303 insertions(+), 17 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 01454abe6..be21f5d63 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -12,7 +12,7 @@ use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::admin::{deny_admin_diagnostic_fallback, handle_admin_eids_lookup}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; @@ -180,6 +180,10 @@ async fn dispatch_fallback( services: &RuntimeServices, mut req: Request, ) -> Result> { + if let Some(response) = deny_admin_diagnostic_fallback(&req) { + return Ok(response); + } + trusted_server_core::integrations::gpt_diagnostics::prepare_request(&state.settings, &mut req)?; let path = req.uri().path().to_string(); let method = req.method().clone(); diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index f0aa4f9bb..3738ef4c7 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -327,6 +327,82 @@ async fn authenticated_admin_eids_route_returns_200() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { + let ec_id = format!("{}.abc123", "a".repeat(64)); + let valid_paths = [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{ec_id}"), + "/_ts/admin/eids".to_owned(), + ]; + + for path in valid_paths { + for method in ["POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"] { + let request = Request::builder() + .method(method) + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(AxumBody::from("sensitive-admin-body")) + .expect("should build authenticated admin request"); + let response = make_service() + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should respond"); + + assert_eq!(response.status().as_u16(), 405); + assert_eq!( + response + .headers() + .get("allow") + .and_then(|v| v.to_str().ok()), + Some("GET") + ); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store") + ); + } + } + + for path in [ + "/_ts/admin/ec/".to_owned(), + format!("/_ts/admin/ec/{ec_id}/extra"), + "/_ts/admin/eids/".to_owned(), + "/_ts/admin/eids/extra".to_owned(), + ] { + for method in ["GET", "POST"] { + let request = Request::builder() + .method(method) + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(AxumBody::from("sensitive-admin-body")) + .expect("should build malformed admin request"); + let response = make_service() + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should respond"); + + assert_eq!(response.status().as_u16(), 404); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store") + ); + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { // Regression for the credential-leak finding: the production basic-auth regex diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index eb2ac2709..4b5e23bcc 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -13,7 +13,7 @@ use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; #[cfg(target_arch = "wasm32")] use trusted_server_core::config_payload::settings_from_config_blob; use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::admin::{deny_admin_diagnostic_fallback, handle_admin_eids_lookup}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; @@ -384,6 +384,9 @@ fn build_router(state: &Arc) -> RouterService { ) -> Result { let services = build_per_request_services(&ctx); let mut req = ctx.into_request(); + if let Some(response) = deny_admin_diagnostic_fallback(&req) { + return Ok(response); + } if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( &state.settings, &mut req, diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index 4631481e2..8cfbfead7 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -323,6 +323,70 @@ async fn authenticated_admin_eids_route_returns_200() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { + let ec_id = format!("{}.abc123", "a".repeat(64)); + let valid_paths = [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{ec_id}"), + "/_ts/admin/eids".to_owned(), + ]; + + for path in valid_paths { + for method in ["POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"] { + let request = request_builder() + .method(method) + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::from("sensitive-admin-body")) + .expect("should build authenticated admin request"); + let response = route(test_router(), request).await; + + assert_eq!(response.status().as_u16(), 405); + assert_eq!( + response + .headers() + .get("allow") + .and_then(|v| v.to_str().ok()), + Some("GET") + ); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store") + ); + } + } + + for path in [ + "/_ts/admin/ec/".to_owned(), + format!("/_ts/admin/ec/{ec_id}/extra"), + "/_ts/admin/eids/".to_owned(), + "/_ts/admin/eids/extra".to_owned(), + ] { + for method in ["GET", "POST"] { + let request = request_builder() + .method(method) + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::from("sensitive-admin-body")) + .expect("should build malformed admin request"); + let response = route(test_router(), request).await; + + assert_eq!(response.status().as_u16(), 404); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store") + ); + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_route_without_credentials_returns_401() { let router = test_router(); diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 85a0eb9d6..0c3232567 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -102,7 +102,9 @@ use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::{handle_admin_ec_lookup, handle_admin_eids_lookup}; +use trusted_server_core::ec::admin::{ + deny_admin_diagnostic_fallback, handle_admin_ec_lookup, handle_admin_eids_lookup, +}; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; @@ -724,6 +726,10 @@ async fn dispatch_fallback( services: &RuntimeServices, mut req: Request, ) -> Response { + if let Some(response) = deny_admin_diagnostic_fallback(&req) { + return response; + } + let path = req.uri().path().to_string(); let method = req.method().clone(); @@ -1819,6 +1825,78 @@ mod tests { } } + #[test] + fn authenticated_admin_diagnostic_fallback_is_denied_locally() { + let router = test_router(); + let ec_id = format!("{}.abc123", "a".repeat(64)); + let valid_paths = [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{ec_id}"), + "/_ts/admin/eids".to_owned(), + ]; + + for path in valid_paths { + for method in [ + Method::POST, + Method::HEAD, + Method::OPTIONS, + Method::PUT, + Method::PATCH, + Method::DELETE, + ] { + let request = request_builder() + .method(method.clone()) + .uri(format!("https://test-publisher.com{path}")) + .header(header::AUTHORIZATION, "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(Body::from("sensitive-admin-body")) + .expect("should build authenticated admin request"); + let response = route(&router, request); + + assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED); + assert_eq!( + response + .headers() + .get(header::ALLOW) + .and_then(|v| v.to_str().ok()), + Some("GET") + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("no-store") + ); + } + } + + for path in [ + "/_ts/admin/ec/".to_owned(), + format!("/_ts/admin/ec/{ec_id}/extra"), + "/_ts/admin/eids/".to_owned(), + "/_ts/admin/eids/extra".to_owned(), + ] { + for method in [Method::GET, Method::POST] { + let request = request_builder() + .method(method) + .uri(format!("https://test-publisher.com{path}")) + .header(header::AUTHORIZATION, "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(Body::from("sensitive-admin-body")) + .expect("should build malformed admin request"); + let response = route(&router, request); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("no-store") + ); + } + } + } + #[test] fn dispatch_identify_options_routes_to_cors_preflight() { // Parity guard: OPTIONS /_ts/api/v1/identify must reach diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 20f89c360..757dac8ff 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -11,7 +11,7 @@ use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::admin::{deny_admin_diagnostic_fallback, handle_admin_eids_lookup}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; @@ -688,6 +688,9 @@ fn build_router(state: &Arc) -> RouterService { ) -> Result { let services = build_runtime_services(&ctx); let mut req = ctx.into_request(); + if let Some(response) = deny_admin_diagnostic_fallback(&req) { + return Ok(response); + } if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( &state.settings, &mut req, diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index b6c5a19ec..777f4057d 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -158,6 +158,70 @@ async fn authenticated_admin_eids_route_returns_200() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { + let ec_id = format!("{}.abc123", "a".repeat(64)); + let valid_paths = [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{ec_id}"), + "/_ts/admin/eids".to_owned(), + ]; + + for path in valid_paths { + for method in ["POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"] { + let request = request_builder() + .method(method) + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::from("sensitive-admin-body")) + .expect("should build authenticated admin request"); + let response = route(test_router(), request).await; + + assert_eq!(response.status().as_u16(), 405); + assert_eq!( + response + .headers() + .get("allow") + .and_then(|v| v.to_str().ok()), + Some("GET") + ); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store") + ); + } + } + + for path in [ + "/_ts/admin/ec/".to_owned(), + format!("/_ts/admin/ec/{ec_id}/extra"), + "/_ts/admin/eids/".to_owned(), + "/_ts/admin/eids/extra".to_owned(), + ] { + for method in ["GET", "POST"] { + let request = request_builder() + .method(method) + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::from("sensitive-admin-body")) + .expect("should build malformed admin request"); + let response = route(test_router(), request).await; + + assert_eq!(response.status().as_u16(), 404); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store") + ); + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn health_route_returns_ok() { // Parity with the Fastly/Axum adapters: GET /health is a cheap liveness probe diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 0e081275b..9d65a9d93 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -76,20 +76,14 @@ fn admin_diagnostic_shape(path: &str) -> Option { /// reaches fallback return `404 Not Found`. Unrelated publisher paths return /// `None` so normal fallback behavior remains unchanged. #[must_use] -pub fn deny_admin_diagnostic_fallback( - req: &Request, -) -> Option> { +pub fn deny_admin_diagnostic_fallback(req: &Request) -> Option> { let shape = admin_diagnostic_shape(req.uri().path())?; - let mut response = if shape == AdminDiagnosticShape::ValidResource - && req.method() != Method::GET - { - json_error(StatusCode::METHOD_NOT_ALLOWED, "method not allowed") - } else { - json_error( - StatusCode::NOT_FOUND, - "admin diagnostic route not found", - ) - }; + let mut response = + if shape == AdminDiagnosticShape::ValidResource && req.method() != Method::GET { + json_error(StatusCode::METHOD_NOT_ALLOWED, "method not allowed") + } else { + json_error(StatusCode::NOT_FOUND, "admin diagnostic route not found") + }; if response.status() == StatusCode::METHOD_NOT_ALLOWED { response From 4657cbf8ac5f536e2c81c3288e7f25d246ebdb27 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 10:32:42 +0530 Subject: [PATCH 298/395] Keep admin EID diagnostics read only --- .../trusted-server-adapter-fastly/src/app.rs | 56 ++++++++++++++++++- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 0c3232567..dcdbb76e2 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -52,7 +52,8 @@ //! `route_request` (tracked in issue #495): //! //! - [`build_ec_request_state`] runs before every dispatched route (except -//! batch-sync, which uses Bearer auth) and reproduces the legacy +//! batch-sync, which uses Bearer auth, and the read-only admin EIDs +//! diagnostic) and reproduces the legacy //! pre-routing prelude: device signals, bot gate, `ts-eids`/`sharedid` //! cookie capture, geo lookup, [`EcContext`] creation, and KV-graph gating. //! - `handle_auction` and integration proxy dispatch receive the same @@ -531,6 +532,17 @@ async fn execute_named( return Ok(run_batch_sync(&state, &services, req)); } + // This diagnostic only previews request cookies. Running the normal EC + // lifecycle would attach finalization state and could ingest those cookies + // into KV after the handler returns, violating the endpoint's read-only + // contract. + if matches!(handler, NamedRouteHandler::AdminEidsLookup) { + let response = PartnerRegistry::from_config(&state.settings.ec.partners) + .and_then(|registry| handle_admin_eids_lookup(®istry, &req)) + .unwrap_or_else(|error| http_error(&error)); + return Ok(response); + } + if let Err(report) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( &state.settings, &mut req, @@ -589,8 +601,7 @@ async fn run_named_route( handle_admin_ec_lookup(kv.as_ref(), &partner_registry, &req) } NamedRouteHandler::AdminEidsLookup => { - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; - handle_admin_eids_lookup(&partner_registry, &req) + unreachable!("admin EIDs lookup should be handled before EC setup") } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::BatchSync => { @@ -1284,6 +1295,7 @@ mod tests { AppState, NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, TrustedServerApp, build_state_from_settings, startup_error_router, }; + use base64::Engine as _; use bytes::Bytes; use edgezero_core::body::Body; use edgezero_core::http::{Method, Response, StatusCode, header, request_builder}; @@ -2182,6 +2194,44 @@ mod tests { ); } + #[test] + fn admin_eids_diagnostic_skips_ec_finalization() { + let router = test_router(); + let ec_id = format!("{}.abc123", "a".repeat(64)); + let eids = serde_json::json!([{ + "source": "example.com", + "uids": [{ "id": "example-uid", "atype": 1 }] + }]); + let eids_cookie = base64::engine::general_purpose::STANDARD.encode(eids.to_string()); + let mut request = request_builder() + .method(Method::GET) + .uri("https://test-publisher.com/_ts/admin/eids") + .header(header::AUTHORIZATION, "Basic YWRtaW46YWRtaW4tcGFzcw==") + .header( + header::COOKIE, + format!("ts-ec={ec_id}; ts-eids={eids_cookie}; sharedId=example-shared-id"), + ) + .body(Body::empty()) + .expect("should build authenticated EIDs diagnostic request"); + request.extensions_mut().insert(DeviceSignals::derive( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + Some("t13d1516h2_8daaf6152771_b186095e22b6"), + Some("1:65536;2:0;4:6291456;6:262144"), + )); + + let response = route(&router, request); + + assert_eq!(response.status(), StatusCode::OK); + assert!( + response + .extensions() + .get::() + .is_none(), + "admin EIDs diagnostics should not attach EC finalization state" + ); + } + #[test] fn dispatch_head_on_named_get_route_falls_through_to_publisher_fallback() { // Regression guard: HEAD /first-party/proxy must reach the publisher From 63e39b8c297d32d51cf5cd363d59c9b7b41d7b5b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 10:35:40 +0530 Subject: [PATCH 299/395] Preserve raw admin EC diagnostic records --- crates/trusted-server-core/src/ec/admin.rs | 173 ++++++++++++++++----- 1 file changed, 136 insertions(+), 37 deletions(-) diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 9d65a9d93..59c8473a1 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -107,23 +107,22 @@ struct AdminEcLookupResponse { /// (`consent.ok = false`). Absent when the body failed to parse. #[serde(skip_serializing_if = "Option::is_none")] tombstone: Option, - /// The stored entry, re-serialized verbatim except for derived + /// The stored entry, preserved as raw JSON except for derived /// `created_iso` / `updated_iso` companions added next to the stored /// unix-seconds timestamps for readability. Absent when the body - /// failed to deserialize (see `entry_error` / `raw_body`). + /// was not valid JSON (see `entry_error` / `raw_body`). #[serde(skip_serializing_if = "Option::is_none")] entry: Option, - /// Deserialization or validation failure detail for the entry body. + /// JSON parsing, schema deserialization, or validation failure detail. #[serde(skip_serializing_if = "Option::is_none")] entry_error: Option, - /// Raw entry body (lossy UTF-8) when it could not be deserialized. + /// Raw entry body (lossy UTF-8) when it was not valid JSON. #[serde(skip_serializing_if = "Option::is_none")] raw_body: Option, - /// The stored KV metadata mirror, when present and parseable. + /// The stored KV metadata JSON, when present and parseable. #[serde(skip_serializing_if = "Option::is_none")] metadata: Option, - /// Deserialization failure detail for the metadata, including its raw - /// value. + /// JSON parsing or schema deserialization failure detail for metadata. #[serde(skip_serializing_if = "Option::is_none")] metadata_error: Option, /// Derived auction view. Present only when the entry deserializes and @@ -268,36 +267,49 @@ fn build_lookup_response( auction: None, }; - match serde_json::from_slice::(&lookup.body) { - Ok(entry) => { - payload.tombstone = Some(!entry.consent.ok); - match entry.validate() { - Ok(()) => payload.auction = Some(build_auction_view(registry, &entry)), - Err(message) => { - payload.entry_error = Some(format!( - "entry failed validation (auction reads fail closed \ - and attach no EIDs): {message}" - )); + match serde_json::from_slice::(&lookup.body) { + Ok(mut entry_json) => { + add_iso_timestamp_companions(&mut entry_json); + payload.entry = Some(entry_json); + + match serde_json::from_slice::(&lookup.body) { + Ok(entry) => { + payload.tombstone = Some(!entry.consent.ok); + match entry.validate() { + Ok(()) => payload.auction = Some(build_auction_view(registry, &entry)), + Err(message) => { + payload.entry_error = Some(format!( + "entry failed validation (auction reads fail closed \ + and attach no EIDs): {message}" + )); + } + } + } + Err(error) => { + payload.entry_error = + Some(format!("failed to deserialize entry schema: {error}")); } } - payload.entry = Some(entry_json_with_iso_timestamps(&entry)); } Err(error) => { - payload.entry_error = Some(format!("failed to deserialize entry: {error}")); + payload.entry_error = Some(format!("failed to parse entry JSON: {error}")); payload.raw_body = Some(String::from_utf8_lossy(&lookup.body).into_owned()); } } match &lookup.metadata { None => {} - Some(bytes) => match serde_json::from_slice::(bytes) { - Ok(metadata) => { - payload.metadata = - Some(serde_json::to_value(&metadata).expect("should serialize KvMetadata")); + Some(bytes) => match serde_json::from_slice::(bytes) { + Ok(metadata_json) => { + payload.metadata = Some(metadata_json); + if let Err(error) = serde_json::from_slice::(bytes) { + payload.metadata_error = + Some(format!("failed to deserialize metadata schema: {error}")); + } } Err(error) => { payload.metadata_error = Some(format!( - "failed to deserialize metadata: {error} (raw: {})", + "failed to parse metadata JSON: {error} (raw: {})", String::from_utf8_lossy(bytes) )); } @@ -307,26 +319,31 @@ fn build_lookup_response( payload } -/// Serializes an entry, adding derived ISO 8601 companions next to the +/// Adds derived ISO 8601 companions next to the /// stored unix-seconds timestamps (`created_iso`, `consent.updated_iso`). /// -/// The stored numeric values stay untouched so the echo remains faithful to -/// what is in KV; the ISO fields exist purely for operator readability. -fn entry_json_with_iso_timestamps(entry: &KvEntry) -> JsonValue { - let mut entry_json = serde_json::to_value(entry).expect("should serialize KvEntry"); - +/// Every stored value, including pre-existing ISO companions, stays untouched. +/// The derived fields exist purely for operator readability when absent. +fn add_iso_timestamp_companions(entry_json: &mut JsonValue) { + let created = entry_json.get("created").and_then(JsonValue::as_u64); + let updated = entry_json + .get("consent") + .and_then(|consent| consent.get("updated")) + .and_then(JsonValue::as_u64); if let Some(object) = entry_json.as_object_mut() { - if let Some(iso) = iso_timestamp(entry.created) { - object.insert("created_iso".to_owned(), JsonValue::String(iso)); + if let Some(iso) = created.and_then(iso_timestamp) { + object + .entry("created_iso".to_owned()) + .or_insert(JsonValue::String(iso)); } if let Some(consent) = object.get_mut("consent").and_then(JsonValue::as_object_mut) - && let Some(iso) = iso_timestamp(entry.consent.updated) + && let Some(iso) = updated.and_then(iso_timestamp) { - consent.insert("updated_iso".to_owned(), JsonValue::String(iso)); + consent + .entry("updated_iso".to_owned()) + .or_insert(JsonValue::String(iso)); } } - - entry_json } /// Formats a unix-seconds timestamp as ISO 8601 (`yyyy-MM-ddTHH:mm:ss.SSSZ`). @@ -588,6 +605,10 @@ mod tests { fn kv_with_raw_body(ec_id: &str, body: &str) -> KvIdentityGraph { let metadata = serde_json::json!({ "ok": true, "country": "US", "v": 1 }).to_string(); + kv_with_raw_body_and_metadata(ec_id, body, &metadata) + } + + fn kv_with_raw_body_and_metadata(ec_id: &str, body: &str, metadata: &str) -> KvIdentityGraph { let store = InMemoryEcKv::new("test-store"); store .insert( @@ -765,6 +786,84 @@ mod tests { ); } + #[test] + fn preserves_raw_entry_and_metadata_shapes() { + let ec_id = test_ec_id(); + let body = serde_json::json!({ + "v": 1, + "created": 1_741_824_000_u64, + "created_iso": "stored-created-iso", + "future_top_level": { "enabled": true }, + "consent": { + "ok": true, + "updated": 1_741_824_000_u64, + "updated_iso": "stored-updated-iso", + "future_consent": "preserve-me" + }, + "geo": { "country": "US" }, + "pub_properties": { + "origin_domain": "example.com", + "seen_domains": { + "example.com": { "first": 1000, "last": 1200, "visits": 3 } + } + }, + "ids": { + "bidstream.example": { "uid": "uid-live", "synced": 1100 } + } + }) + .to_string(); + let metadata = serde_json::json!({ + "ok": true, + "country": "US", + "v": 1, + "future_metadata": { "source": "edge" } + }) + .to_string(); + let kv = kv_with_raw_body_and_metadata(&ec_id, &body, &metadata); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + let json = response_json(response); + + assert_eq!(json["entry"]["future_top_level"]["enabled"], true); + assert_eq!(json["entry"]["consent"]["future_consent"], "preserve-me"); + assert_eq!(json["entry"]["ids"]["bidstream.example"]["synced"], 1100); + assert!( + json["entry"]["pub_properties"]["seen_domains"].is_object(), + "legacy map-shaped seen_domains should remain unchanged" + ); + assert_eq!(json["entry"]["created_iso"], "stored-created-iso"); + assert_eq!( + json["entry"]["consent"]["updated_iso"], + "stored-updated-iso" + ); + assert_eq!(json["metadata"]["future_metadata"]["source"], "edge"); + assert_eq!(json["auction"]["eids"][0]["source"], "bidstream.example"); + } + + #[test] + fn valid_json_with_invalid_entry_schema_remains_visible() { + let ec_id = test_ec_id(); + let body = serde_json::json!({ "future": "value" }).to_string(); + let kv = kv_with_raw_body(&ec_id, &body); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + let json = response_json(response); + + assert_eq!(json["entry"]["future"], "value"); + assert!( + json["entry_error"] + .as_str() + .expect("should have entry_error") + .contains("failed to deserialize entry schema") + ); + assert!(json.get("raw_body").is_none()); + assert!(json.get("auction").is_none()); + } + #[test] fn reports_tombstone_entries() { let ec_id = test_ec_id(); @@ -829,7 +928,7 @@ mod tests { json["entry_error"] .as_str() .expect("should have entry_error") - .contains("failed to deserialize"), + .contains("failed to parse entry JSON"), "should describe the parse failure" ); assert_eq!(json["raw_body"], "not json at all"); From 178e2cde08969468c9f19388e1b001c2e556c1df Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 10:36:59 +0530 Subject: [PATCH 300/395] Document admin EC and EID diagnostics --- docs/guide/api-reference.md | 65 +++++++++++++++++++ ...8-admin-diagnostics-review-fixes-design.md | 2 +- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 340d5d425..2a30a1ac4 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -7,6 +7,7 @@ Quick reference for all Trusted Server HTTP endpoints. - [First-Party Endpoints](#first-party-endpoints) - Core ad serving and proxying - [Edge Cookie Endpoints](#edge-cookie-endpoints) - Identity sync and enrichment - [Request Signing](#request-signing-endpoints) - Cryptographic signing and key management +- [Admin Diagnostics](#admin-diagnostic-endpoints) - Protected EC troubleshooting - [TSJS Library](#tsjs-library-endpoint) - JavaScript library serving - [Utility Endpoints](#utility-endpoints) - Optional operational helpers - [Integration Endpoints](#integration-endpoints) - Third-party service proxying @@ -580,6 +581,67 @@ curl -X POST https://edge.example.com/_ts/admin/keys/deactivate \ --- +## Admin Diagnostic Endpoints + +These endpoints expose sensitive identity and cookie data and require HTTP Basic Authentication. Configure a handler that covers the entire `/_ts/admin` namespace; startup rejects configurations that do not protect every admin route. All responses are JSON with `Cache-Control: no-store`. + +The examples below use fictional IDs and values only. + +### GET /\_ts/admin/ec + +### GET /\_ts/admin/ec/`{id}` + +Reads an EC identity-graph record for troubleshooting. The explicit route accepts an EC ID in `{64 lowercase hex}.{6 alphanumeric}` format. The bare route uses the request's `ts-ec` cookie. + +This lookup is implemented only by the Fastly adapter because the identity graph is stored in Fastly KV. Other adapters return `501 Not Implemented`. + +**Response fields:** + +- `ec_id`, `store`, and `generation` identify the raw KV lookup. +- `entry` preserves the stored JSON shape, including unknown and legacy fields. Derived `created_iso` and `consent.updated_iso` fields are added only when absent. +- `metadata` preserves the stored metadata JSON shape. +- `tombstone` reports whether consent has been withdrawn. +- `auction.eids` previews the partner EIDs the stored record can contribute; `auction.skipped` explains filtered IDs. +- `entry_error`, `metadata_error`, and `raw_body` keep malformed or schema-incompatible records inspectable. + +The auction preview validates the stored record and partner configuration, but cannot reproduce live per-request consent checks. It must not be treated as proof that a specific auction request will receive those EIDs. + +**Status codes:** + +| Status | Meaning | +| ------ | ----------------------------------------------------------- | +| `200` | Record found, including inspectable corrupt records | +| `400` | Invalid explicit EC ID | +| `401` | Missing or invalid Basic credentials | +| `404` | Record not found, or the bare route has no `ts-ec` cookie | +| `405` | Method other than `GET` (`Allow: GET`) | +| `501` | EC identity graph unavailable on this adapter or deployment | + +```bash +curl -u admin:secure-password \ + "https://edge.example.com/_ts/admin/ec/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.abc123" + +curl -u admin:secure-password \ + --cookie "ts-ec=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.abc123" \ + "https://edge.example.com/_ts/admin/ec" +``` + +### GET /\_ts/admin/eids + +Parses the request's `ts-eids` and `sharedId` cookies and previews which configured partner IDs cookie ingestion would match or drop. It performs request inspection only: it does not read or write KV and is available on every adapter. + +After successful authentication this endpoint always returns `200 OK`; missing or malformed cookies are represented by `cookie_present`, `sharedid_present`, and `parse_error`. The `ingest.matched` and `ingest.unmatched` arrays show the ingestion preview. + +```bash +curl -u admin:secure-password \ + --cookie "sharedId=fictional-shared-id" \ + "https://edge.example.com/_ts/admin/eids" +``` + +Malformed diagnostic paths return a local `404`, and unsupported methods return a local `405`; they are never forwarded to the publisher origin. + +--- + ## TSJS Library Endpoint ### GET /static/tsjs=`` @@ -768,6 +830,9 @@ curl -u admin:secure-password https://edge.example.com/_ts/admin/keys/rotate - `/_ts/admin/keys/rotate` - `/_ts/admin/keys/deactivate` +- `/_ts/admin/ec` +- `/_ts/admin/ec/{id}` +- `/_ts/admin/eids` - Any paths matching configured `handlers` patterns --- diff --git a/docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md b/docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md index f6cf8aa06..b77b5732b 100644 --- a/docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md +++ b/docs/superpowers/specs/2026-08-18-admin-diagnostics-review-fixes-design.md @@ -108,7 +108,7 @@ authentication: - A non-GET request to `/_ts/admin/ec`, a single-segment `/_ts/admin/ec/{id}`, or `/_ts/admin/eids` returns local `405 Method Not - Allowed` with `Allow: GET`. +Allowed` with `Allow: GET`. - A path with a trailing slash, an extra segment, a missing segment structure, or an EIDs suffix returns local `404 Not Found` and never reaches the publisher. From c70de71cd0ac31ad1b3524d8fe920913e8d1d839 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 10:41:05 +0530 Subject: [PATCH 301/395] Fix admin diagnostic test lint --- crates/trusted-server-core/src/ec/admin.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 59c8473a1..f3f3bb18f 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -615,7 +615,7 @@ mod tests { ec_id, EcKvWrite { body, - metadata: &metadata, + metadata, ttl: Duration::from_secs(60), mode: EcKvWriteMode::Add, }, From 69e1b8aa65bdbf82ecf2864575393077e5480de4 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 10:50:07 +0530 Subject: [PATCH 302/395] Reserve the full admin fallback namespace --- crates/trusted-server-adapter-axum/src/app.rs | 16 ++---- .../tests/routes.rs | 15 ++++++ .../src/app.rs | 17 ++---- .../tests/routes.rs | 15 ++++++ .../trusted-server-adapter-fastly/src/app.rs | 3 ++ crates/trusted-server-adapter-spin/src/app.rs | 17 ++---- .../tests/routes.rs | 15 ++++++ crates/trusted-server-core/src/ec/admin.rs | 53 ++++++++++++++++--- 8 files changed, 107 insertions(+), 44 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index be21f5d63..a96ba9e3c 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -12,7 +12,9 @@ use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::{deny_admin_diagnostic_fallback, handle_admin_eids_lookup}; +use trusted_server_core::ec::admin::{ + admin_ec_lookup_not_supported, deny_admin_diagnostic_fallback, handle_admin_eids_lookup, +}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; @@ -438,17 +440,7 @@ fn named_route_handler( NamedRouteHandler::AdminEcNotSupported => { // The EC identity graph is Fastly KV backed; the Axum // dev server has no store to read. - let body = edgezero_core::body::Body::from( - "Admin EC lookup is not supported on the Axum dev server.\n\ - Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", - ); - let mut resp = Response::new(body); - *resp.status_mut() = StatusCode::NOT_IMPLEMENTED; - resp.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - Ok(resp) + Ok(admin_ec_lookup_not_supported()) } NamedRouteHandler::AdminEidsLookup => { let partner_registry = diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index 3738ef4c7..bb4204ff9 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -299,6 +299,18 @@ async fn authenticated_admin_ec_routes_return_501() { 501, "{path} should report that Axum EC lookup is unsupported" ); + assert_eq!( + resp.headers() + .get("content-type") + .and_then(|v| v.to_str().ok()), + Some("application/json") + ); + assert_eq!( + resp.headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store") + ); } } @@ -375,6 +387,9 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { format!("/_ts/admin/ec/{ec_id}/extra"), "/_ts/admin/eids/".to_owned(), "/_ts/admin/eids/extra".to_owned(), + "/_ts/admin/eids.json".to_owned(), + "/_ts/admin/ec;foo".to_owned(), + format!("/_ts/admin/ec%2F{ec_id}"), ] { for method in ["GET", "POST"] { let request = Request::builder() diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 4b5e23bcc..cade09a4e 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -13,7 +13,10 @@ use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; #[cfg(target_arch = "wasm32")] use trusted_server_core::config_payload::settings_from_config_blob; use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::{deny_admin_diagnostic_fallback, handle_admin_eids_lookup}; +use trusted_server_core::ec::admin::{ + admin_ec_lookup_not_supported as core_admin_ec_lookup_not_supported, + deny_admin_diagnostic_fallback, handle_admin_eids_lookup, +}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; @@ -252,17 +255,7 @@ fn admin_key_management_not_supported() -> Response { } fn admin_ec_lookup_not_supported() -> Response { - let body = edgezero_core::body::Body::from( - "Admin EC lookup is not supported on Cloudflare Workers.\n\ - Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", - ); - let mut response = Response::new(body); - *response.status_mut() = StatusCode::NOT_IMPLEMENTED; - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response + core_admin_ec_lookup_not_supported() } /// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index 8cfbfead7..c512e2e9c 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -301,6 +301,18 @@ async fn authenticated_admin_ec_routes_return_501() { 501, "{path} should report that Cloudflare EC lookup is unsupported" ); + assert_eq!( + resp.headers() + .get("content-type") + .and_then(|v| v.to_str().ok()), + Some("application/json") + ); + assert_eq!( + resp.headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store") + ); } } @@ -365,6 +377,9 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { format!("/_ts/admin/ec/{ec_id}/extra"), "/_ts/admin/eids/".to_owned(), "/_ts/admin/eids/extra".to_owned(), + "/_ts/admin/eids.json".to_owned(), + "/_ts/admin/ec;foo".to_owned(), + format!("/_ts/admin/ec%2F{ec_id}"), ] { for method in ["GET", "POST"] { let request = request_builder() diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index dcdbb76e2..9a8ea1a3a 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1887,6 +1887,9 @@ mod tests { format!("/_ts/admin/ec/{ec_id}/extra"), "/_ts/admin/eids/".to_owned(), "/_ts/admin/eids/extra".to_owned(), + "/_ts/admin/eids.json".to_owned(), + "/_ts/admin/ec;foo".to_owned(), + format!("/_ts/admin/ec%2F{ec_id}"), ] { for method in [Method::GET, Method::POST] { let request = request_builder() diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 757dac8ff..77aaa0567 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -11,7 +11,10 @@ use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::{deny_admin_diagnostic_fallback, handle_admin_eids_lookup}; +use trusted_server_core::ec::admin::{ + admin_ec_lookup_not_supported as core_admin_ec_lookup_not_supported, + deny_admin_diagnostic_fallback, handle_admin_eids_lookup, +}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; @@ -367,17 +370,7 @@ fn admin_key_management_not_supported() -> Response { } fn admin_ec_lookup_not_supported() -> Response { - let body = edgezero_core::body::Body::from( - "Admin EC lookup is not supported on Fermyon Spin.\n\ - Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", - ); - let mut response = Response::new(body); - *response.status_mut() = StatusCode::NOT_IMPLEMENTED; - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response + core_admin_ec_lookup_not_supported() } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 777f4057d..68ac4d55a 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -136,6 +136,18 @@ async fn authenticated_admin_ec_routes_return_501() { 501, "{path} should report that Spin EC lookup is unsupported" ); + assert_eq!( + resp.headers() + .get("content-type") + .and_then(|v| v.to_str().ok()), + Some("application/json") + ); + assert_eq!( + resp.headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store") + ); } } @@ -200,6 +212,9 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { format!("/_ts/admin/ec/{ec_id}/extra"), "/_ts/admin/eids/".to_owned(), "/_ts/admin/eids/extra".to_owned(), + "/_ts/admin/eids.json".to_owned(), + "/_ts/admin/ec;foo".to_owned(), + format!("/_ts/admin/ec%2F{ec_id}"), ] { for method in ["GET", "POST"] { let request = request_builder() diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index f3f3bb18f..46170c53a 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -64,7 +64,14 @@ fn admin_diagnostic_shape(path: &str) -> Option { }); } - path.starts_with("/_ts/admin/eids/") + if path.starts_with("/_ts/admin/eids/") { + return Some(AdminDiagnosticShape::Malformed); + } + + // Reserve the complete admin namespace at the publisher-fallback boundary. + // A successfully authenticated malformed or future admin path must never + // forward its Authorization header or body to the publisher origin. + (path == "/_ts/admin" || path.starts_with("/_ts/admin/")) .then_some(AdminDiagnosticShape::Malformed) } @@ -72,9 +79,9 @@ fn admin_diagnostic_shape(path: &str) -> Option { /// an adapter's publisher fallback. /// /// Valid diagnostic resources reject non-GET methods with `405 Method Not -/// Allowed`. Malformed, trailing, and any valid GET route that unexpectedly -/// reaches fallback return `404 Not Found`. Unrelated publisher paths return -/// `None` so normal fallback behavior remains unchanged. +/// Allowed`. Malformed, trailing, unknown, and any valid GET admin route that +/// unexpectedly reaches fallback return `404 Not Found`. Paths outside the +/// reserved `/_ts/admin` namespace return `None`, preserving normal fallback. #[must_use] pub fn deny_admin_diagnostic_fallback(req: &Request) -> Option> { let shape = admin_diagnostic_shape(req.uri().path())?; @@ -178,10 +185,7 @@ pub fn handle_admin_ec_lookup( req: &Request, ) -> Result, Report> { let Some(kv) = kv else { - return Ok(json_error( - StatusCode::NOT_IMPLEMENTED, - "EC identity graph is not configured on this deployment", - )); + return Ok(admin_ec_lookup_not_supported()); }; let ec_id = match requested_ec_id(req) { @@ -207,6 +211,15 @@ pub fn handle_admin_ec_lookup( Ok(json_response(StatusCode::OK, body)) } +/// Returns the portable response used when an adapter has no EC KV backend. +#[must_use] +pub fn admin_ec_lookup_not_supported() -> Response { + json_error( + StatusCode::NOT_IMPLEMENTED, + "EC identity graph is not configured on this deployment", + ) +} + /// Resolves the EC ID to look up from the path or the `ts-ec` cookie. /// /// Returns the (boxed) error response to send directly when no valid ID is @@ -692,6 +705,10 @@ mod tests { format!("/_ts/admin/ec/{ec_id}/extra"), "/_ts/admin/eids/".to_owned(), "/_ts/admin/eids/extra".to_owned(), + "/_ts/admin/eids.json".to_owned(), + "/_ts/admin/ec;foo".to_owned(), + format!("/_ts/admin/ec%2F{ec_id}"), + "/_ts/admin/unknown".to_owned(), ]; for path in paths { @@ -1022,6 +1039,26 @@ mod tests { assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED); } + #[test] + fn unsupported_ec_lookup_response_is_json_and_no_store() { + let response = admin_ec_lookup_not_supported(); + + assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED); + assert_eq!( + response.headers().get(header::CONTENT_TYPE), + Some(&HeaderValue::from_static("application/json")) + ); + assert_eq!( + response.headers().get(header::CACHE_CONTROL), + Some(&HeaderValue::from_static("no-store")) + ); + assert!( + response_json(response)["error"] + .as_str() + .is_some_and(|message| message.contains("not configured")) + ); + } + #[test] fn kv_read_failure_propagates() { let kv = KvIdentityGraph::failing("broken-store"); From 9841fcc9b901ae53371605260959f77537206575 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 10:55:27 +0530 Subject: [PATCH 303/395] Collect the root page on every device profile The multi-profile crawl repeated the first collector for the root page instead of the profile being walked, so every later profile recorded the root as the first device. On a site whose root offers no crawl targets that hid the device split entirely, and the first profile's literal GAM unit path was written as if both devices agreed with it. --- .../src/commands/audit/generate/mod.rs | 63 ++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 2a2d4b11b..454abb16d 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -561,7 +561,7 @@ pub(crate) fn run_update_slots( // one page, which inference already refuses to represent. for (index, (label, collector)) in collectors.iter().enumerate() { if index > 0 { - let repeat = first_collector.collect_page(&root_url, request.cookies); + let repeat = collector.collect_page(&root_url, request.cookies); match repeat { Ok(page) => fold_collected(&mut table, &root_url, &page, &mut notes)?, Err(error) => notes.push(format!("skipped `{root_url}` on {label}: {error}")), @@ -1715,6 +1715,67 @@ mod tests { .expect("a slot without an explicit unit path must still load"); } + #[test] + fn a_root_only_site_is_still_collected_on_every_device_profile() { + // A site whose root offers no crawl targets is audited on the root page + // alone. If the later profiles never load it, a device split there is + // invisible and the first profile's literal path gets written as if + // every device agreed with it. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&config_path, loadable_config()).expect("should write config"); + + let desktop = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/desktop/homepage", + &[], + ), + )]); + let mobile = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/mobile/homepage", + &[], + ), + )]); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + ) + .expect("the run should complete and report the conflict"); + + assert_eq!( + mobile.visited.borrow().as_slice(), + ["https://publisher.example/"], + "the mobile profile must load the root even when there is nothing else to crawl" + ); + let written = fs::read_to_string(&config_path).expect("read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert!( + value["creative_opportunities"]["slot"][0] + .get("gam_unit_path") + .is_none(), + "a root-only device split must not write either device's literal path, got:\n{written}" + ); + trusted_server_core::settings::Settings::from_toml(&written) + .expect("a slot without an explicit unit path must still load"); + } + #[test] fn a_crawl_refuses_when_most_pages_are_challenged() { // Bot protection serves an interstitial that loads fine and has no ad From 8f5b3457e255b453d135eaeeff371023130ae05f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 12:06:16 +0530 Subject: [PATCH 304/395] Design SSAT debug comment output formatting --- ...-08-18-ssat-debug-comment-format-design.md | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-18-ssat-debug-comment-format-design.md diff --git a/docs/superpowers/specs/2026-08-18-ssat-debug-comment-format-design.md b/docs/superpowers/specs/2026-08-18-ssat-debug-comment-format-design.md new file mode 100644 index 000000000..0cb6f5ec0 --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-ssat-debug-comment-format-design.md @@ -0,0 +1,143 @@ +# SSAT Debug Comment Output Format Design + +**Date:** 2026-08-18 + +**Status:** Approved for implementation planning + +**Related PR:** [IABTechLab/trusted-server#943](https://github.com/IABTechLab/trusted-server/pull/943) + +## Summary + +The configurable SSAT `ts-debug` comment currently serializes its auction dump +as compact JSON. Full-verbosity dumps can contain bids, creatives, provider +metadata, and PBS HTTP-call diagnostics, making the single-line payload hard to +navigate in page source. + +Add a `format` option to `[debug.auction_html_comment_options]` with two values: + +- `compact` (default): preserve the existing one-line JSON representation. +- `pretty`: serialize the outer auction dump as indented JSON. + +Pretty formatting does not parse or transform JSON-looking strings such as PBS +`requestbody` and `responsebody`. Those values remain byte-for-byte equivalent +JSON string values so the comment continues to represent what the provider +integration captured rather than a guessed interpretation of it. + +## Goals + +1. Make the outer provider, bid, metadata, and HTTP-call hierarchy easier to + navigate in HTML source during local debugging. +2. Preserve today's compact output unless an operator explicitly opts in. +3. Preserve the existing dump schema and the exact contents of nested strings. +4. Keep the existing security and size protections independent of formatting. + +## Non-goals + +- Recursively parsing or formatting `requestbody`, `responsebody`, creative + markup, or any other string that happens to contain JSON. +- Adding a browser UI, CLI viewer, downloadable artifact, or syntax highlighting. +- Changing verbosity, redaction, section toggles, metadata selection, creative + truncation, or provider response capture. +- Making the 256 KiB total dump cap configurable. + +## Configuration + +Extend `AuctionDebugCommentOptions` with an enum-backed field: + +```rust +#[serde(default)] +pub format: AuctionDebugCommentFormat, + +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuctionDebugCommentFormat { + #[default] + Compact, + Pretty, +} +``` + +The hand-written `Default` implementation for `AuctionDebugCommentOptions` +sets `format` to `Compact`, matching serde's default. Unknown strings fail +configuration loading because serde rejects enum variants other than `compact` +and `pretty`; the enum deliberately has no catch-all variant. + +Example: + +```toml +[debug.auction_html_comment_options] +include_provider_responses = true +include_mediator_response = false +include_bids = false +metadata_keys = ["error_type", "http_status", "message"] +verbosity = "full" +format = "pretty" +``` + +`trusted-server.example.toml` documents both accepted values and keeps +`format = "compact"` in the example to make the default visible. + +## Rendering and Data Flow + +`prepend_auction_debug_comment` continues building the same +`serde_json::Value::Object`. Only the serialization selection changes: + +- `Compact` uses `serde_json::to_string`. +- `Pretty` uses `serde_json::to_string_pretty`. + +The serialized string then passes through the existing processing in the same +order: + +1. Neutralize HTML comment terminators (`-->` and `--!>`). +2. Apply the unconditional 256 KiB total serialized dump cap. +3. Place the result after `dump=` in the `ts-debug` HTML comment. + +Pretty mode intentionally introduces line breaks after `dump=`. The comment's +summary line remains unchanged, and both modes remain valid JSON whenever the +total cap is not reached. + +Pretty printing increases serialized size, so it may reach the existing cap +sooner than compact output. If capped, the dump may end mid-JSON exactly as it +can today; the existing `…(truncated N bytes)` marker continues to make that +condition explicit. + +## Compatibility and Safety + +- Existing configurations that omit `format` produce byte-for-byte-equivalent + compact JSON output. +- Formatting does not change which fields are included or expose data hidden by + `redacted` or `upstream` verbosity. +- Nested request and response bodies remain strings. Pretty mode adds whitespace + only to the outer serialized representation; deserializing an uncapped dump + produces the same JSON value as compact mode. +- Comment-terminator neutralization and the total cap apply in both modes. +- The existing warning remains: `upstream` and `full` output can expose identity + and request data and must not be enabled in production. + +## Testing + +Follow test-driven development with focused tests before production changes: + +1. Settings deserialization accepts `format = "pretty"` and defaults to + `Compact` when omitted. +2. An invalid format string fails deserialization. +3. Compact rendering retains the existing one-line `dump={...}` representation. +4. Pretty rendering contains indented outer JSON and deserializes to the same + value as compact rendering for the same auction result. +5. A nested JSON `requestbody` remains a JSON string rather than becoming an + object in pretty mode. +6. Pretty rendering still neutralizes every tested HTML-comment terminator. +7. Pretty rendering still respects the 256 KiB total dump cap and emits the + existing truncation marker. + +After focused tests pass, run the repository-required Fastly/core test suite, +format check, and target-matched clippy checks relevant to the changed Rust +core code. + +## Future Direction + +If operators need decoded navigation of nested provider bodies, add a separate +local inspection tool that extracts the comment and conditionally parses known +JSON body fields for display. Keeping that transformation outside the emitted +comment preserves raw capture fidelity and avoids increasing every debug page's +payload. From 056594703bf33f9629fc7aaa4a899269221feaef Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 12:17:37 +0530 Subject: [PATCH 305/395] Plan SSAT debug comment output formatting --- .../2026-08-18-ssat-debug-comment-format.md | 229 ++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md diff --git a/docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md b/docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md new file mode 100644 index 000000000..8a4d8c4dd --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md @@ -0,0 +1,229 @@ +# SSAT Debug Comment Output Format Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a backward-compatible `compact`/`pretty` configuration option for the outer JSON in SSAT auction debug comments. + +**Architecture:** Extend `AuctionDebugCommentOptions` with a serde-backed format enum that defaults to compact. Keep the existing dump value and safety pipeline unchanged, selecting only `serde_json::to_string` versus `serde_json::to_string_pretty` before terminator neutralization and the 256 KiB cap. + +**Tech Stack:** Rust 2024, serde, serde_json, TOML configuration, existing `trusted-server-core` unit tests. + +--- + +## File Map + +- Modify `crates/trusted-server-core/src/settings.rs`: define the output-format enum, add the option and default, and test TOML behavior. +- Modify `crates/trusted-server-core/src/publisher.rs`: select compact or pretty JSON serialization and test rendering/safety invariants. +- Modify `trusted-server.example.toml`: document the new setting and accepted values. + +### Task 1: Add the configuration type + +**Files:** +- Modify: `crates/trusted-server-core/src/settings.rs:1945-2030` +- Test: `crates/trusted-server-core/src/settings.rs:2710-2780` + +- [ ] **Step 1: Write failing settings tests** + +Extend `auction_debug_comment_options_default_matches_serde_defaults` with: + +```rust +assert_eq!( + opts.format, + AuctionDebugCommentFormat::Compact, + "should default to compact output" +); +``` + +Add focused tests: + +```rust +#[test] +fn auction_debug_comment_options_deserializes_pretty_format() { + let options: AuctionDebugCommentOptions = toml::from_str(r#"format = "pretty""#) + .expect("should deserialize pretty format"); + assert_eq!(options.format, AuctionDebugCommentFormat::Pretty); +} + +#[test] +fn auction_debug_comment_options_bad_format_fails_config_load() { + let result: Result = + toml::from_str(r#"format = "expanded""#); + assert!( + result.is_err(), + "unrecognized format must fail to deserialize, not silently fall back" + ); +} +``` + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +cargo test-fastly auction_debug_comment_options -- --nocapture +``` + +Expected: compilation fails because `AuctionDebugCommentFormat` and `format` do not exist. + +- [ ] **Step 3: Implement the minimal configuration surface** + +Add `format` after `verbosity` in `AuctionDebugCommentOptions`: + +```rust +/// JSON representation used for the outer auction dump. +#[serde(default)] +pub format: AuctionDebugCommentFormat, +``` + +Set `format: AuctionDebugCommentFormat::Compact` in the hand-written default and define: + +```rust +/// JSON representation used for the outer `ts-debug` auction dump. +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuctionDebugCommentFormat { + #[default] + Compact, + Pretty, +} +``` + +- [ ] **Step 4: Run the focused tests and verify GREEN** + +Run `cargo test-fastly auction_debug_comment_options -- --nocapture`. + +Expected: all matching settings tests pass. + +- [ ] **Step 5: Commit the configuration change** + +```bash +git add crates/trusted-server-core/src/settings.rs +git commit -m "Configure SSAT debug comment output format" +``` + +### Task 2: Render pretty outer JSON without transforming values + +**Files:** +- Modify: `crates/trusted-server-core/src/publisher.rs:2027-2130` +- Test: `crates/trusted-server-core/src/publisher.rs:4330-4945` + +- [ ] **Step 1: Add a test helper for extracting an uncapped dump** + +Extract the existing parsing logic into `dump_json_from_comment`, returning the dump substring and parsed `serde_json::Value`. It must split after `dump=` and before the comment's final newline/terminator so it works for both one-line and indented JSON. + +- [ ] **Step 2: Write failing rendering tests** + +Add tests proving: + +1. Default compact output still contains `dump={"provider_responses":` and no newline immediately after the opening object. +2. Pretty output contains `dump={\n "provider_responses":`. +3. Compact and pretty uncapped dumps deserialize to equal JSON values. +4. In Full mode, metadata containing `{"requestbody": "{\"id\":\"request-1\"}"}` retains `requestbody` as a JSON string in pretty output. + +Extend the existing comment-terminator test to iterate over both +`AuctionDebugCommentFormat::{Compact, Pretty}` for the tested verbosity modes. +Add a pretty/full case to the total-cap test and assert the existing +`(truncated` marker remains present. + +- [ ] **Step 3: Run the rendering tests and verify RED** + +Run: + +```bash +cargo test-fastly auction_debug_comment -- --nocapture +``` + +Expected: the pretty-layout assertion fails because rendering still always uses compact serialization. + +- [ ] **Step 4: Implement format-selected serialization** + +Import `AuctionDebugCommentFormat` alongside the existing options types. Replace the single serializer call with: + +```rust +let serialized = match options.format { + AuctionDebugCommentFormat::Compact => serde_json::to_string(&dump), + AuctionDebugCommentFormat::Pretty => serde_json::to_string_pretty(&dump), +}; +let dump = render_dump( + serialized.unwrap_or_else(|error| format!("")), +); +``` + +Do not alter the dump value, nested strings, neutralization, cap, or comment envelope. + +- [ ] **Step 5: Run the focused tests and verify GREEN** + +Run `cargo test-fastly auction_debug_comment -- --nocapture`. + +Expected: all matching rendering tests pass. + +- [ ] **Step 6: Commit the renderer change** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Pretty print SSAT debug comment dumps" +``` + +### Task 3: Document and verify the completed feature + +**Files:** +- Modify: `trusted-server.example.toml:180-190` + +- [ ] **Step 1: Document the setting** + +Add to `[debug.auction_html_comment_options]`: + +```toml +# "compact" (default) or "pretty". Pretty formats only the outer dump; +# JSON request/response bodies remain strings exactly as captured. +format = "compact" +``` + +- [ ] **Step 2: Run formatting** + +Run `cargo fmt --all` and then `cargo fmt --all -- --check`. + +Expected: both exit successfully. + +- [ ] **Step 3: Run required tests** + +Run each command separately: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: every command exits successfully with zero failed tests. + +- [ ] **Step 4: Run target-matched lint checks** + +Run each command separately: + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: every command exits successfully with warnings denied. + +- [ ] **Step 5: Inspect the final diff and configuration compatibility** + +Run `git diff --check` and inspect `git diff origin/main...HEAD` plus the remaining working-tree diff. Confirm compact is the serde/default value, pretty changes whitespace only, nested strings are preserved, and both safety protections remain unconditional. + +- [ ] **Step 6: Commit documentation and any formatting changes** + +```bash +git add crates/trusted-server-core/src/settings.rs crates/trusted-server-core/src/publisher.rs trusted-server.example.toml +git commit -m "Document SSAT debug comment formatting" +``` + +- [ ] **Step 7: Push the branch and update PR #943** + +Run `git push origin feat/ssat-debug-comment-config`, then verify PR state and checks with `gh pr view 943` and `gh pr checks 943`. From 2050f1888151112dea27173bb2c40baa1fdaf233 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 12:19:03 +0530 Subject: [PATCH 306/395] Configure SSAT debug comment output format --- crates/trusted-server-core/src/settings.rs | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 783677f4b..9839159d9 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1984,6 +1984,10 @@ pub struct AuctionDebugCommentOptions { /// request/response data may become visible via view-source. #[serde(default)] pub verbosity: AuctionDebugCommentVerbosity, + + /// JSON representation used for the outer auction dump. + #[serde(default)] + pub format: AuctionDebugCommentFormat, } impl Default for AuctionDebugCommentOptions { @@ -1994,6 +1998,7 @@ impl Default for AuctionDebugCommentOptions { include_bids: true, metadata_keys: default_auction_debug_metadata_keys(), verbosity: AuctionDebugCommentVerbosity::Redacted, + format: AuctionDebugCommentFormat::Compact, } } } @@ -2020,6 +2025,15 @@ pub enum AuctionDebugCommentVerbosity { Full, } +/// JSON representation used for the outer `ts-debug` auction dump. +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuctionDebugCommentFormat { + #[default] + Compact, + Pretty, +} + /// Tester-cookie endpoint configuration. #[derive(Debug, Default, Clone, Deserialize, Serialize)] pub struct TesterCookieConfig { @@ -2733,6 +2747,11 @@ mod tests { AuctionDebugCommentVerbosity::Redacted, "should default to Redacted" ); + assert_eq!( + opts.format, + AuctionDebugCommentFormat::Compact, + "should default to compact output" + ); } #[test] @@ -2759,6 +2778,23 @@ mod tests { assert_eq!(options.verbosity, AuctionDebugCommentVerbosity::Upstream); } + #[test] + fn auction_debug_comment_options_deserializes_pretty_format() { + let options: AuctionDebugCommentOptions = toml::from_str(r#"format = "pretty""#) + .expect("should deserialize pretty format"); + assert_eq!(options.format, AuctionDebugCommentFormat::Pretty); + } + + #[test] + fn auction_debug_comment_options_bad_format_fails_config_load() { + let result: Result = + toml::from_str(r#"format = "expanded""#); + assert!( + result.is_err(), + "unrecognized format must fail to deserialize, not silently fall back" + ); + } + #[test] fn bad_verbosity_string_fails_config_load() { // Deserialize AuctionDebugCommentOptions directly, not a full Settings — From 00a969a6a057c58acd67c143e9074a7d1bb076ce Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 12:21:17 +0530 Subject: [PATCH 307/395] Pretty print SSAT debug comment dumps --- crates/trusted-server-core/src/publisher.rs | 147 +++++++++++++++----- crates/trusted-server-core/src/settings.rs | 4 +- 2 files changed, 112 insertions(+), 39 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index f808ea85b..75b5c0d2d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -63,7 +63,7 @@ use crate::response_privacy::enforce_synthesized_html_cache_privacy; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::{ AUCTION_DEBUG_METADATA_ALLOWLIST, AUCTION_DEBUG_UPSTREAM_METADATA_KEYS, - AuctionDebugCommentOptions, AuctionDebugCommentVerbosity, Settings, + AuctionDebugCommentFormat, AuctionDebugCommentOptions, AuctionDebugCommentVerbosity, Settings, }; use crate::streaming_processor::{ BodyStreamDecoder, BodyStreamEncoder, Compression, GzipDecodeReader, PipelineConfig, @@ -2112,10 +2112,13 @@ pub(crate) fn prepend_auction_debug_comment( } }; // Single serialize → single neutralise → single total-budget cap. - let dump = render_dump( - serde_json::to_string(&serde_json::Value::Object(dump)) - .unwrap_or_else(|e| format!("")), - ); + let dump = serde_json::Value::Object(dump); + let serialized = match options.format { + AuctionDebugCommentFormat::Compact => serde_json::to_string(&dump), + AuctionDebugCommentFormat::Pretty => serde_json::to_string_pretty(&dump), + }; + let dump = + render_dump(serialized.unwrap_or_else(|error| format!(""))); let debug_comment = format!( "") .expect("should contain comment terminator"); - let dump: serde_json::Value = + let value: serde_json::Value = serde_json::from_str(dump).expect("should contain valid untruncated JSON"); + (dump, value) + } + + fn response_metadata_from_comment(comment: &str) -> serde_json::Value { + let (_, dump) = dump_from_comment(comment); dump["provider_responses"][0]["metadata"].clone() } @@ -4419,6 +4427,59 @@ mod tests { ); } + #[test] + fn auction_debug_comment_pretty_formats_outer_json_without_changing_value() { + let compact_comment = dump_comment_for_creative("
plain
"); + let pretty_options = AuctionDebugCommentOptions { + format: AuctionDebugCommentFormat::Pretty, + ..AuctionDebugCommentOptions::default() + }; + let pretty_comment = + dump_comment_for_creative_with_options("
plain
", &pretty_options); + + assert!( + compact_comment.contains("dump={\"provider_responses\":"), + "default output should remain compact: {compact_comment}" + ); + assert!( + pretty_comment.contains("dump={\n \"provider_responses\":"), + "pretty output should indent the outer dump: {pretty_comment}" + ); + + let (_, compact_dump) = dump_from_comment(&compact_comment); + let (_, pretty_dump) = dump_from_comment(&pretty_comment); + assert_eq!( + pretty_dump, compact_dump, + "formatting should not change the dump value" + ); + } + + #[test] + fn auction_debug_comment_pretty_preserves_nested_json_as_string() { + let metadata = std::collections::HashMap::from([( + "debug".to_string(), + serde_json::json!({ + "httpcalls": { + "openx": [{ "requestbody": "{\"id\":\"request-1\"}" }] + } + }), + )]); + let options = AuctionDebugCommentOptions { + verbosity: AuctionDebugCommentVerbosity::Full, + format: AuctionDebugCommentFormat::Pretty, + ..AuctionDebugCommentOptions::default() + }; + + let comment = dump_comment_for_metadata_with_options(metadata, &options); + let response_metadata = response_metadata_from_comment(&comment); + let request_body = &response_metadata["debug"]["httpcalls"]["openx"][0]["requestbody"]; + assert_eq!(request_body, "{\"id\":\"request-1\"}"); + assert!( + request_body.is_string(), + "nested request body should remain a string" + ); + } + #[test] fn auction_debug_comment_never_leaks_provider_debug_metadata() { // A provider response whose `debug` metadata mirrors the shape prebid @@ -4827,16 +4888,22 @@ mod tests { #[test] fn verbosity_full_still_hits_overall_byte_cap() { let huge_creative = "z".repeat(MAX_AUCTION_DEBUG_DUMP_BYTES * 2); - let options = AuctionDebugCommentOptions { - verbosity: AuctionDebugCommentVerbosity::Full, - ..AuctionDebugCommentOptions::default() - }; - let comment = dump_comment_for_creative_with_options(&huge_creative, &options); - assert!( - comment.contains("(truncated"), - "even Full verbosity must respect the total dump byte cap: {}", - &comment[..comment.len().min(200)] - ); + for format in [ + AuctionDebugCommentFormat::Compact, + AuctionDebugCommentFormat::Pretty, + ] { + let options = AuctionDebugCommentOptions { + verbosity: AuctionDebugCommentVerbosity::Full, + format, + ..AuctionDebugCommentOptions::default() + }; + let comment = dump_comment_for_creative_with_options(&huge_creative, &options); + assert!( + comment.contains("(truncated"), + "even Full verbosity must respect the total dump byte cap for {format:?}: {}", + &comment[..comment.len().min(200)] + ); + } } #[test] @@ -4913,27 +4980,33 @@ mod tests { AuctionDebugCommentVerbosity::Redacted, AuctionDebugCommentVerbosity::Full, ] { - let options = AuctionDebugCommentOptions { - verbosity, - ..AuctionDebugCommentOptions::default() - }; - for creative in [ - "
evil-->break
", - "--!>", - "", - "", + for format in [ + AuctionDebugCommentFormat::Compact, + AuctionDebugCommentFormat::Pretty, ] { - let comment = dump_comment_for_creative_with_options(creative, &options); - assert_eq!( - comment.matches("-->").count(), - 1, - "exactly one terminator must survive for {verbosity:?}, {creative:?}: {comment}" - ); - assert!( - !comment.contains("--!>"), - "nested terminator must not survive for {verbosity:?}, {creative:?}: {comment}" - ); + let options = AuctionDebugCommentOptions { + verbosity, + format, + ..AuctionDebugCommentOptions::default() + }; + for creative in [ + "
evil-->break
", + "--!>", + "", + "", + ] { + let comment = dump_comment_for_creative_with_options(creative, &options); + assert_eq!( + comment.matches("-->").count(), + 1, + "exactly one terminator must survive for {verbosity:?}, {format:?}, {creative:?}: {comment}" + ); + assert!( + !comment.contains("--!>"), + "nested terminator must not survive for {verbosity:?}, {format:?}, {creative:?}: {comment}" + ); + } } } } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 9839159d9..27d6f6b8e 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2780,8 +2780,8 @@ mod tests { #[test] fn auction_debug_comment_options_deserializes_pretty_format() { - let options: AuctionDebugCommentOptions = toml::from_str(r#"format = "pretty""#) - .expect("should deserialize pretty format"); + let options: AuctionDebugCommentOptions = + toml::from_str(r#"format = "pretty""#).expect("should deserialize pretty format"); assert_eq!(options.format, AuctionDebugCommentFormat::Pretty); } From e4c52a8ada0b23d3ef9d5099004efe7c52ce91a3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 12:24:18 +0530 Subject: [PATCH 308/395] Document SSAT debug comment formatting --- trusted-server.example.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trusted-server.example.toml b/trusted-server.example.toml index debdbdfa1..1cbd6f91c 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -187,6 +187,9 @@ metadata_keys = ["error_type", "http_status", "message"] # request or identity data. "full" additionally exposes all response metadata # and untruncated creatives. Never use either sensitive mode in production. verbosity = "redacted" +# "compact" (default) or "pretty". Pretty formats only the outer dump; +# JSON request/response bodies remain strings exactly as captured. +format = "compact" [creative_opportunities] gam_network_id = "123456789" From ccda5208f72fe7f201cfe221af4000ca278d4470 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 12:30:12 +0530 Subject: [PATCH 309/395] Format SSAT debug comment implementation plan --- docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md b/docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md index 8a4d8c4dd..bbe214363 100644 --- a/docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md +++ b/docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md @@ -19,6 +19,7 @@ ### Task 1: Add the configuration type **Files:** + - Modify: `crates/trusted-server-core/src/settings.rs:1945-2030` - Test: `crates/trusted-server-core/src/settings.rs:2710-2780` @@ -104,6 +105,7 @@ git commit -m "Configure SSAT debug comment output format" ### Task 2: Render pretty outer JSON without transforming values **Files:** + - Modify: `crates/trusted-server-core/src/publisher.rs:2027-2130` - Test: `crates/trusted-server-core/src/publisher.rs:4330-4945` @@ -167,6 +169,7 @@ git commit -m "Pretty print SSAT debug comment dumps" ### Task 3: Document and verify the completed feature **Files:** + - Modify: `trusted-server.example.toml:180-190` - [ ] **Step 1: Document the setting** From 67d1608317cff58e9bb91c4329b81209c4883570 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 12:37:12 +0530 Subject: [PATCH 310/395] Document SSAT auction debug comment usage --- docs/guide/auction-orchestration.md | 69 +++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index b4ba6b797..6fa5cc9f3 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -823,3 +823,72 @@ Every auction response includes structured metadata in `ext.orchestrator`: "time_ms": 145 } ``` + +### SSAT HTML Debug Comment + +For local server-side auction template (SSAT) investigation, Trusted Server can +insert a `` comment before the page's bids script. Enable +it in `trusted-server.toml`, push the local configuration, restart the local +server, and search the page source for `ts-debug`: + +```toml +[debug] +auction_html_comment = true + +[debug.auction_html_comment_options] +include_provider_responses = true +include_mediator_response = false +include_bids = false +metadata_keys = ["error_type", "http_status", "message"] +verbosity = "full" +format = "pretty" +``` + +```bash +ts config validate +ts config push --adapter fastly --local +fastly compute serve +``` + +This example is useful when investigating raw Prebid Server requests and +responses without spending the dump budget on winning creatives. Raw PBS +`debug.httpcalls` and `resolvedrequest` metadata also require +`debug = true` under `[integrations.prebid]`. + +| Option | Default | Behavior | +| ---------------------------- | -------------------------------------- | ----------------------------------------------------------------------- | +| `include_provider_responses` | `true` | Include the provider response array | +| `include_mediator_response` | `true` | Include the mediator response when a mediator ran | +| `include_bids` | `true` | Include bid objects; when `false`, provider status and metadata remain | +| `metadata_keys` | `error_type`, `http_status`, `message` | Select a subset of the fixed validated metadata keys in `redacted` mode | +| `verbosity` | `redacted` | Select `redacted`, `upstream`, or `full` sensitivity | +| `format` | `compact` | Use compact outer JSON or indented outer JSON with `pretty` | + +The verbosity modes form an explicit sensitivity ladder: + +- `redacted` reconstructs only validated `error_type`, `http_status`, and a + server-generated `message`, intersected with `metadata_keys`. A successful + provider response can therefore have `metadata: {}`. +- `upstream` adds provider-controlled errors, warnings, response timings, bid + statuses, and bounded upstream-message fields. It does not include raw PBS + `httpcalls` or `resolvedrequest`. +- `full` includes raw response metadata and untruncated creatives. It can expose + IP addresses, geo data, identifiers, consent strings, request signatures, and + complete provider request/response bodies. + +`format = "pretty"` indents only the outer dump. JSON-looking fields such as +`requestbody` and `responsebody` remain strings exactly as captured, so their +contents still appear escaped. Use a local JSON inspection tool when those +nested values need additional formatting. + +The summary line's `winning=N` count is computed before section filtering, so +it can be nonzero while `include_bids = false` produces empty bid arrays. Every +mode and format neutralizes HTML-comment terminators and enforces a 256 KiB +total dump cap. A capped dump ends with `…(truncated N bytes)` and is no longer +valid JSON. + +::: danger Local debugging only +Do not enable the auction HTML comment in production. Even `redacted` can +contain bid-level data and creative previews, while `upstream` and `full` may +expose identity-bearing request data to anyone who can view the page source. +::: From cd2419264edf7296b16d2aee9938df8f1b8bc22b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 13:33:15 +0530 Subject: [PATCH 311/395] Document PR 823 review resolution design --- ...6-08-18-pr-823-review-resolution-design.md | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md diff --git a/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md new file mode 100644 index 000000000..477662ccb --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md @@ -0,0 +1,216 @@ +# PR 823 Review Resolution Design + +## Goal + +Resolve the actionable findings in review `4958563121` on PR 823 without +unrelated refactoring, verify the complete branch, publish the fixes, and reply +to every inline review thread with concrete resolution evidence. + +## Scope + +The implementation covers all 28 inline threads and all actionable items in the +review summary. The summary's explicitly out-of-scope pre-existing +partially-invalid `page_patterns` behavior is not expanded into this PR unless a +fix is required by another in-scope change. The PR description's stale legacy +alias sentence is corrected after the branch changes are published. + +Each reviewer suggestion is verified against the current code. A suggestion is +implemented when it is correct for this repository. Where repository evidence +contradicts a suggestion, the implementation retains the correct behavior and +the review response explains the evidence. + +## Design Principles + +- Preserve operator-authored configuration, comments, ordering, and unrelated + sections byte-for-byte wherever possible. +- Never print secrets or whole effective configuration documents as diagnostic + output. +- Never turn uncertain crawl evidence into a runnable fabricated ad-unit path. +- Treat browser navigation as a session, not a sequence of isolated launches. +- Keep `generate`, `verify`, static CLI commands, and runtime matching on shared + domain rules instead of parallel reimplementations. +- Bound all page-controlled data and browser operations. +- Use test-first changes for behavior corrections and minimal annotations for + code-quality-only corrections. + +## Component Design + +### 1. Configuration integrity and command output + +`slot_toml` will replace the line-oriented slot-boundary heuristic with a +TOML-aware edit strategy. The resulting document must preserve every top-level +item outside the managed creative-opportunity fields and preserve comments +adjacent to or between operator sections. Non-contiguous slot declarations, +multiline values, arrays whose continuation lines begin with `[`, trailing +comments, CRLF input, and inline-slot conversion receive regression coverage. +The updater will reject a candidate if preservation cannot be proven. + +Generation will re-read the source config immediately before the atomic write +and refuse to overwrite a concurrently edited file. `--dry-run` will emit only +the managed creative-opportunities change, never the complete config. Notes and +rollback warnings go to stderr so machine-readable stdout remains clean. Tests +will prove that dry-run leaves the source file byte-identical and does not expose +unrelated secret-bearing keys. + +Merge behavior remains add-only for operator-authored data: existing templated +unit paths are retained, newly observed formats are unioned, and multiple +discovered placements absorbed by one broad configured div prefix produce an +operator note. + +### 2. Crawl evidence and inference + +Inference will preserve evidence instead of silently collapsing it: + +- Non-ASCII shared-prefix computation uses UTF-8 byte boundaries. +- Same-page normalization collisions retain distinct raw placements and emit a + diagnostic rather than silently dropping formats. Numeric-only stable tokens + are not classified as hexadecimal hash noise. +- Multi-slot SRA request fallbacks are ignored when `dids` names more than one + slot. +- A page is considered empty only when no audited profile found slots there. +- Fragment detection requires stronger evidence: a useful shared prefix, or at + least three disjoint fragments. Ambiguous two-slot groups are retained with a + note. +- Locale landing paths are emitted literally when they are shorter than the + inferred section depth, and literal path segments are escaped before being + interpolated into globs. +- Refused template decisions are omitted from generated slots and surfaced with + their reasons. The documentation and tests will consistently describe these + cases as refusal, not literal fallback. +- The redundant witness rule is removed or made independently meaningful. The + actual crawler will support the section depth that inference can produce; + locale-prefixed behavior will not exist only in hand-built evidence tests. +- Dropped-section diagnostics are capped, percent-encoded paths are normalized + before filtering, and page-like extensions are classified consistently. + +The root page and section pages for a device profile are collected in one +browser session. Page analysis that parses full HTML is moved off the +current-thread CDP event pump. Each page/tab is closed on every success and +error path. + +### 3. Shared browser behavior + +The browser collectors will share executable discovery and launch/session +configuration. Browser options exposed to operators will have one meaning in +`page`, `verify`, and `generate`: Chrome override, settling, headful/headless +mode, device profile/viewport, proxy, consent assumption, cookies, and TLS +policy. + +`verify` will reuse one browser/runtime/profile across its URLs so clearance and +session state survive. The generic/legacy generator will default to the same +consent assumption as ad-template generation and expose the opt-out rather than +depending on `derive(Default)`. + +Cookie parameters are explicitly host-only with `Path=/`. A same-host +`http`-to-`https` upgrade is accepted with a redirect note; host changes, +downgrades, and unexpected port changes remain cross-origin refusals. Failure to +read or parse the final browser URL fails closed instead of substituting the +requested URL. + +Every post-navigation evaluation is time-bounded. The collector enlarges the +resource timing buffer before navigation, waits for an interactive or complete +document before accruing quiet time, honors sub-poll quiet windows, validates +`quiet <= max`, and reports saturation. Navigation load-event timeout is a +warning after a successful `goto`; it does not discard readable page evidence. +Evidence payload bytes and captured string lengths are capped before expensive +decode/allocation. + +Init-script and page-evaluation failures become explicit warnings or errors +rather than empty evidence. Promise-returning sitemap evaluation awaits its +result. Main-frame-only collection is disclosed when frames are skipped. + +The injected collector will be behavior-preserving: size pairs enforce the +`u32` range, the `googletag` setter is total, the unused non-variadic `cmd.push` +wrapper is removed, wrapping markers are closure-local/non-enumerable, and +page-derived warning text is terminal-safe. + +### 4. Runtime and static-command parity + +Expected-slot projection uses the runtime's renderability rule. Slots the +runtime omits for a path do not count as matched verification slots; diagnostics +state that the runtime omits the slot on that path rather than claiming the +whole config is rejected. + +Configured media type remains a typed `MediaType` through comparison and is +rendered to a string only at the output boundary. Slots that the phase-one +checker cannot confirm (video/native-only and out-of-page) are represented as +unconfirmable and do not fail `--strict`; genuinely partial or missing +confirmable slots still fail. Slot phase is absent when no evidence exists. +The server-side APS compatibility field no longer creates unconditional +client-side `fetchBids` warnings. + +Collector warnings are included in page results. Human output includes the +runtime expectation, gate summary, matched count, extra evidence, and warnings +already present in JSON. Output escaping covers Unicode bidi controls and all +config-derived strings. + +`explain` reports exactly the shared runtime gate result. Provider configuration +is a separate advisory. The unsupported `--edgezero-enabled` model and stale +legacy-fallback claim are removed because no runtime condition backs them. +Gate diagnostics consume the shared gate result instead of rebuilding lists by +hand. The hot runtime gate avoids heap allocation, the seven-boolean wrapper is +removed, and the consent tri-state is documented and exhaustively tested. + +`compile_page_pattern` becomes crate-private and a public validation-only API is +used by the CLI. Specific compile failures are retained in logs. HTTP methods +use `http::Method` parsing so CLI semantics match the runtime. + +### 5. CLI contracts, documentation, and CI + +Clap owns argument validation: URL parsing happens at the value parser, the +audit namespace uses help-on-missing-subcommand, `check` uses an argument group +and conflicts, and settle bounds are rejected during parsing. Parser tests cover +the visible command shapes and legacy restrictions. + +CI-oriented assertion failures exit 1; tool/configuration/navigation failures +exit 2. Assertion text is written directly and cannot disappear behind a log +filter. The guide documents all four `ts config ad-templates` commands, all +flags, shared config-loading flags, browser flags, consent/profile behavior, +dry-run output, and exit codes. + +Browser fixture CI either installs/resolves Chrome and requires the tests to +execute, or explicitly opts into a mode that fails when Chrome is unavailable; +it may not report success after silently skipping every browser assertion. + +All real-looking customer identifiers and names introduced by this PR are +replaced with fictional values in tests, comments, and documentation. Stale +module-level lint suppressions, inaccurate docs, assertion messages, enum +ordering, dead query matching, and orphaned comments are corrected without +unrelated cleanup. + +## Error Handling and Compatibility + +All new Rust fallible paths use the repository's existing `CliResult` / +`error-stack` conventions. Browser failures identify the operation and URL but +do not include cookies, configuration values, or page payloads. Best-effort +cleanup must not replace an earlier collection error. + +JSON compatibility is preserved where possible. New distinctions are additive +or correct semantically invalid fields: unconfirmable status is explicit, and +phase may be omitted when there was no evidence. Documentation is updated with +the exact wire behavior. + +## Verification Strategy + +Each behavioral issue follows red-green-refactor: + +1. Add the smallest unit, parser, orchestration, or fixture test reproducing the + review finding. +2. Run the narrow test and confirm the expected failure. +3. Implement the minimal correction. +4. Re-run the narrow test and the affected crate suite. + +Final verification runs the repository-required commands relevant to the +changed surface: CLI tests through `scripts/test-cli.sh`, target-matched Rust +tests, JS tests when the collector script changes, `cargo fmt --all -- --check`, +all target-matched clippy aliases, documentation formatting, and browser fixture +tests with an available Chrome. Any environment-dependent test that cannot run +is reported explicitly and is not described as passing. + +## Review Replies and Publication + +Changes are grouped into reviewable commits by component, then pushed to the PR +branch after final verification. Each inline reply is posted in its existing +thread and states the concrete change, relevant test, or evidence-backed reason +for retaining behavior. Replies avoid generic acknowledgements. Threads are not +replied to as fixed until the corresponding commit is visible on GitHub. From 4b779ce3de8a54760a8b8653f7596d609802fabc Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 13:34:58 +0530 Subject: [PATCH 312/395] Clarify PR review resolution design --- ...6-08-18-pr-823-review-resolution-design.md | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md index 477662ccb..d17a09302 100644 --- a/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md +++ b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md @@ -152,8 +152,16 @@ hand. The hot runtime gate avoids heap allocation, the seven-boolean wrapper is removed, and the consent tri-state is documented and exhaustively tested. `compile_page_pattern` becomes crate-private and a public validation-only API is -used by the CLI. Specific compile failures are retained in logs. HTTP methods -use `http::Method` parsing so CLI semantics match the runtime. +used by the CLI. `lint` explicitly reports every configured page pattern the +runtime would drop, while the broader pre-existing runtime acceptance policy +remains out of scope. Specific compile failures are retained in logs. HTTP +methods use `http::Method` parsing so CLI semantics match the runtime. + +Full URLs and bare path inputs pass through the same URL normalization rules: +percent-encoding, dot-segment resolution, query/fragment removal, and leading +slash behavior must be identical. Scheme detection is anchored to the path +portion before `?`, so an absolute URL inside a query value does not cause a +bare path to be parsed as a full URL. ### 5. CLI contracts, documentation, and CI @@ -178,6 +186,34 @@ module-level lint suppressions, inaccurate docs, assertion messages, enum ordering, dead query matching, and orphaned comments are corrected without unrelated cleanup. +## Inline Review Traceability + +| Thread | Resolution area | +| --- | --- | +| `3802056460`, `3802056470` | TOML-aware splice and comment/value preservation | +| `3802056474` | Secret-safe dry-run and stderr diagnostics | +| `3802056481` | Omit and explain refused slots | +| `3802056488` | UTF-8-safe div prefix calculation | +| `3802056494` | Same-page normalized-div collisions | +| `3802056497` | Locale landing-page patterns | +| `3802056502` | Multi-profile empty-page accounting | +| `3802056508` | Close every browser tab | +| `3802056513` | Enforce JavaScript-to-Rust `u32` bounds | +| `3802056521`, `3802056529` | Total GPT hook and removal of behavior-changing `cmd.push` wrapper | +| `3802056539` | Shared faithful browser launch configuration | +| `3802056549`, `3802056555` | Correct settling and load-timeout handling | +| `3802056559` | Preserve injected collector warnings | +| `3802056564`, `3802056571` | Runtime renderability parity and accurate diagnostics | +| `3802056580`, `3802056584` | Unconfirmable status and removal of false APS warning | +| `3802056586` | Identical URL and bare-path normalization | +| `3802056593` | Fictional committed examples | +| `3802056599` | Browser fixture CI must execute or fail loudly | +| `3802056605` | Add-only merge of formats with broad-prefix diagnostics | +| `3802056614` | Consent parity for generic and legacy generation | +| `3802056623` | Refusal behavior, tests, and documentation agree | +| `3802056628` | Safe same-host HTTP-to-HTTPS redirect handling | +| `3802056638` | Remove ungrounded EdgeZero fallback model | + ## Error Handling and Compatibility All new Rust fallible paths use the repository's existing `CliResult` / From f3cb0104c4f1507c64f041f21eed5b14dfc64d87 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 13:52:58 +0530 Subject: [PATCH 313/395] Plan PR 823 review resolution --- .../2026-08-18-pr-823-review-resolution.md | 712 ++++++++++++++++++ 1 file changed, 712 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md diff --git a/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md new file mode 100644 index 000000000..703fc3978 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md @@ -0,0 +1,712 @@ +# PR 823 Review Resolution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve every actionable finding in PR 823 review `4958563121`, verify the branch, publish it, and answer all 28 inline threads. + +**Architecture:** Correct the review findings at four existing seams: core runtime gate APIs, pure CLI projection/comparison, crawl generation and TOML persistence, and the shared browser session. Keep page-controlled work bounded, use one source of truth for runtime/browser behavior, and preserve operator-authored configuration outside the managed creative-opportunities fields (`slot`, `gam_network_id`, `section_root`, and `section_segment`). + +**Tech Stack:** Rust 2024, clap 4, toml_edit 0.23, chromiumoxide 0.9, Tokio current-thread runtime, serde/serde_json, embedded JavaScript collector, mdBook documentation, GitHub CLI. + +--- + +## File Map + +- `crates/trusted-server-core/src/creative_opportunities.rs`: allocation-free gate evaluation, gate diagnostics, pattern validation, consent semantics. +- `crates/trusted-server-core/src/publisher.rs`: named gate input at the runtime call site. +- `crates/trusted-server-cli/src/ad_templates/{expected,compare,output}.rs`: runtime-equivalent projection, typed formats, confirmability, safe output. +- `crates/trusted-server-cli/src/commands/config/ad_templates.rs`: static command validation, gate parity, lint, escaping. +- `crates/trusted-server-cli/src/commands/audit/{collector,browser,ad_templates,ad_template_collector.js}.rs`: shared browser options/session and verifier behavior. +- `crates/trusted-server-cli/src/commands/audit/generate/{browser_collector,evidence,gpt_slots,crawl_plan,page_patterns,unit_template,slot_toml,mod,validate}.rs`: crawl evidence, inference, persistence, and dry-run safety. +- `crates/trusted-server-cli/src/commands/audit/{mod,page}.rs`, `crates/trusted-server-cli/src/run.rs`, `crates/trusted-server-cli/src/main.rs`: clap contracts and exit outcomes. +- `docs/guide/cli.md`, `scripts/test-cli.sh`, `.github/workflows/test.yml`: operator contract and enforced browser CI. + +## Task 1: Make the runtime gate API allocation-free and reusable + +**Files:** +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Add failing core tests** + +Add tests that sweep all 64 boolean combinations with `consent_allows_auction: None`, assert the expected `No`/`Unknown` result, assert `blocking_gates()` derives diagnostics without an owned `Vec`, and exercise the specific page-pattern validation error. + +Use a borrowed/static iterator contract: + +```rust +pub fn blocking_gates(self) -> impl Iterator { + AdStackGateName::ALL + .into_iter() + .filter(move |gate| gate.blocks(self.input)) +} + +pub fn validate_page_pattern(pattern: &str) -> Result<(), String> { + compile_page_pattern(pattern).map(|_| ()) +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-core --target "$(rustc -vV | awk '/host:/ {print $2}')" ad_stack_gate -- --nocapture +``` + +Expected: failure because the unknown-consent sweep and allocation-free diagnostic API are not implemented. + +- [ ] **Step 3: Implement the minimal core change** + +Store the original `AdStackGateInput` in `AdStackGateResult`, compute `expected` with boolean expressions rather than `Vec::push`, expose a zero-allocation iterator over a `const ALL`, make `compile_page_pattern` crate-private, and add `validate_page_pattern`. Document that `None` means unknown and differs from denied (`Some(false)`). Preserve the detailed glob error in `compile_patterns`. + +Delete `should_run_server_side_ad_stack`; construct `AdStackGateInput` with named fields in `publisher.rs`. Import the gate types at module scope. + +- [ ] **Step 4: Verify GREEN** + +Run the narrow command again, then: + +```bash +cargo test-fastly creative_opportunities +cargo test-axum creative_opportunities +``` + +Expected: all selected tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/publisher.rs +git commit -m "Align ad stack gate diagnostics with runtime" +``` + +## Task 2: Align expected-slot projection and comparison with runtime behavior + +**Files:** +- Modify: `crates/trusted-server-cli/src/ad_templates/expected.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/compare.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_templates.rs` + +- [ ] **Step 1: Add failing projection and comparison tests** + +Cover: + +- an unrenderable dynamic slot is omitted from expected slots and does not make `matched_slots` pass; +- the diagnostic says the runtime omits the slot for that path; +- `MediaType` remains typed through comparison; +- video/native-only and out-of-page slots produce `Unconfirmable` and do not fail strict; +- an incompatible banner is still `Partial` and fails strict; +- a missing slot has `phase: None` and JSON omits `phase`; +- server-side APS configuration alone does not emit `aps_evidence_missing`; +- collector warnings are appended to page warnings; +- human output contains expectation, gates, matched count, extra evidence, and warnings; +- bidi override/isolate characters are escaped. + +The central type changes are: + +```rust +pub struct ExpectedFormat { + pub width: u32, + pub height: u32, + pub media_type: MediaType, +} + +pub enum SlotStatus { + Confirmed, + Partial, + Missing, + Unconfirmable, +} + +pub struct SlotResult { + pub phase: Option, + // existing fields +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" ad_templates::expected +cargo test --package trusted-server-cli --target "$HOST_TARGET" ad_templates::compare +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::ad_templates +``` + +Expected: new assertions fail on current projection/status/warning behavior. + +- [ ] **Step 3: Implement projection, comparison, and output changes** + +Filter `match_slots` with `render_gam_unit_path(...).map(...)` while building `ExpectedSlot`. Remove the unconditional client-side APS check. Compute confirmability before assigning status. Map typed media values to strings only in `to_slot_json`. Make JSON phase `Option` with `skip_serializing_if = "Option::is_none"`. Extend warnings with `evidence.warnings` after decode. + +Extend `is_terminal_control` with `0x202A..=0x202E` and `0x2066..=0x2069`. Apply `escape_terminal_text` to every human-facing page/config-derived field. + +- [ ] **Step 4: Verify GREEN** + +Run all three narrow commands again. + +Expected: all selected tests pass with no warnings. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/ad_templates crates/trusted-server-cli/src/commands/audit/ad_templates.rs +git commit -m "Match ad template verification to runtime behavior" +``` + +## Task 3: Correct static CLI contracts and process exit semantics + +**Files:** +- Modify: `crates/trusted-server-cli/src/commands/config/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` +- Modify: `crates/trusted-server-cli/src/main.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `Cargo.lock` + +- [ ] **Step 1: Add failing parser, normalization, lint, and outcome tests** + +Add tests proving: + +- bare and full-URL forms normalize spaces, dot segments, tabs, queries, and fragments identically; +- `/r?to=https://example.com` remains a bare path; +- `check` requires exactly one expectation mode and rejects `--allow-extra-slots --expect-no-slots` through clap; +- `--method` accepts a valid `http::Method` and uses exact GET semantics; +- `lint` reports each invalid configured pattern; +- `explain` uses `gate.expected` even when providers are empty and prints provider state separately; +- `--edgezero-enabled` is rejected because the unsupported model is removed; +- bare `ts audit` displays help rather than a drifting manual error; +- parser coverage includes lint, explain, generate, verify profiles/options, and the no-`--adapter` contract; +- an assertion outcome maps to exit 1 and a tool error maps to exit 2. + +Use an explicit process outcome: + +```rust +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RunOutcome { + Success, + AssertionFailed, +} + +impl RunOutcome { + pub const fn exit_code(self) -> i32 { + match self { + Self::Success => 0, + Self::AssertionFailed => 1, + } + } +} +``` + +Tool failures remain `Err(String)` and therefore exit 2. Assertion commands write their failure to stderr before returning `AssertionFailed`, avoiding `log::error!` filtering. + +- [ ] **Step 2: Run parser/static tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::config::ad_templates +cargo test --package trusted-server-cli --target "$HOST_TARGET" run::tests +``` + +Expected: current hand-rolled validation, normalization, and exit behavior fail the new tests. + +- [ ] **Step 3: Implement the CLI contract** + +Use a dummy HTTPS base with `Url::options().base_url(...)` for bare paths after anchored scheme detection on the pre-query slice. Add clap `ArgGroup`, `conflicts_with`, `arg_required_else_help`, typed `http::Method`, and browser settle validation. Add `http = { workspace = true }` to the CLI host dependencies. + +Return `RunOutcome` from dispatchable CI commands. Keep edgezero delegated errors as tool errors. Remove the unsupported EdgeZero flag/text and route gate output through `blocking_gates()`. + +- [ ] **Step 4: Verify GREEN** + +Run the two narrow commands again and confirm all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add Cargo.lock crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/main.rs crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/commands/config/ad_templates.rs +git commit -m "Define ad template CLI assertion contracts" +``` + +## Task 4: Make the injected collector bounded and behavior-preserving + +**Files:** +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_template_collector.js` +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` + +- [ ] **Step 1: Add failing JavaScript-contract and decoder tests** + +Add tests/fixtures for an out-of-`u32` size beside a valid slot, a truthy `googletag.cmd` without `push`, multiple `cmd.push` arguments, 512-character capture limits, and non-enumerable/closure-local wrapping. Replace the existing `contains("cmd.push")` assertion with assertions that the no-op wrapper is absent. + +The JavaScript bounds are: + +```javascript +const __TS_MAX_STRING = 512 +function __ts_text(value) { + return String(value).slice(0, __TS_MAX_STRING) +} + +if (width > 4294967295 || height > 4294967295) return null +``` + +The setter must always retain the publisher value: + +```javascript +set(value) { + try { + internal = wrap(value) + } catch (error) { + internal = value + __ts_push(__ts_ev.warnings, { + code: "wrap_failed", + message: __ts_text(error), + }) + } +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::collector +cargo test --package trusted-server-cli --target "$HOST_TARGET" collector_payload +``` + +Expected: current script permits oversized integers and retains the behavior-changing wrapper. + +- [ ] **Step 3: Implement minimal collector changes** + +Guard all page-derived strings through `__ts_text`, enforce numeric upper bounds, delete the `cmd.push` wrapper, use a closure-local `WeakSet` for wrapped objects, and install wrapped functions with non-enumerable `Object.defineProperty`. Soften the header claim to “observes without capturing page data.” + +Before serde decode, stringify the evidence inside the page and return a small +sentinel instead of the payload when the serialized string exceeds 1 MiB +(`MAX_EVIDENCE_PAYLOAD_BYTES = 1_048_576`). On the Rust side, the sentinel +produces an `ad_evidence_too_large` warning and `ad_evidence: None`; it does not +fail navigation or the whole collection. This bounds CDP transfer and Rust +decode/allocation while preserving a precise operator diagnostic. + +- [ ] **Step 4: Verify GREEN** + +Run the narrow commands again and confirm all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/ad_template_collector.js crates/trusted-server-cli/src/commands/audit/collector.rs crates/trusted-server-cli/src/commands/audit/browser.rs +git commit -m "Bound browser ad template evidence collection" +``` + +## Task 5: Unify browser launch, session reuse, and settling + +**Files:** +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/page.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing fake-collector and browser configuration tests** + +Cover one launch/session for multiple URLs, root included in the profile batch, page close on success/error, host-only `Path=/` cookies, explicit final-URL failure, same-host HTTP-to-HTTPS acceptance, host/downgrade/port refusal, new-headless 1280x800 defaults, headful/profile/proxy/consent parity, `$CHROME` parity, and generic/legacy default-on consent. + +Extend the trait with a default batch method so fakes remain simple: + +```rust +pub trait AuditCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result; + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + requests.iter().cloned().map(|request| self.collect_page(request)).collect() + } +} +``` + +The real browser implementation overrides `collect_pages` to create one runtime, +temporary profile, browser, handler, and sequentially closed pages. + +- [ ] **Step 2: Run narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::browser +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::ad_templates +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::browser_collector +``` + +Expected: verifier launches per URL, browser defaults diverge, and tabs/cookies/final URL handling fail new assertions. + +- [ ] **Step 3: Implement shared browser configuration and batching** + +Move executable resolution and launch-option construction into `browser.rs` as crate-visible helpers used by both collectors. Flatten shared browser options into generate and verify, while keeping generation-only pacing/crawl flags local. Build cookies with explicit domain from `url.host_str()` and `path = Some("/".to_string())`; do not set `url` simultaneously. + +In each page collector, capture the inner result, always call bounded `page.close().await`, then return the captured result. Batch verify requests via `collect_pages`. Include the root in each profile's batch rather than collecting it in a throwaway session. Use `spawn_blocking` for scraper analysis before folding results. + +- [ ] **Step 4: Bound post-navigation work and correct settle semantics** + +Install `performance.setResourceTimingBufferSize(100000)` before navigation. Make `settle` return warnings and wrap every `evaluate`, URL/title read, scroll operation, and evidence read in a per-operation timeout. Accrue quiet only after `document.readyState` is `interactive` or `complete`; sleep `min(remaining_quiet, 250ms)` so short quiet values are honored. Treat `wait_for_navigation` timeout as a warning after successful `goto`. + +Propagate GPT/link/sitemap evaluation errors as notes, set `await_promise` for sitemap discovery, and warn when only the main frame is inspected while child frames exist. + +- [ ] **Step 5: Verify GREEN** + +Run all three narrow commands again. If Chrome is available, also run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::browser::tests:: -- --ignored --test-threads=1 +``` + +Expected: unit/fake tests pass; browser fixtures execute and pass when Chrome exists. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit +git commit -m "Share browser sessions across ad template audits" +``` + +## Task 6: Preserve crawl evidence and make inference conservative + +**Files:** +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing inference tests** + +Add focused tests for: + +- `annonsü1`/`annonsü2` and `ünicode-ad-a`/`ünicode-ad-b` prefixes; +- desktop-empty/mobile-present and the inverse; +- two disjoint unrelated placements retained, two with a useful prefix or three fragments refused; +- same-page normalized UUID collisions retained with raw div IDs and all formats; +- 16+ digit numeric stable segments retained; +- comma-separated SRA `dids` ignored; +- locale `/en` pattern emitted as `/en` and every emitted glob matches its source path; +- glob metacharacters escaped with `glob::Pattern::escape`; +- percent-encoded noise/extension paths and `.html`/`.htm`/`.php` treatment; +- dropped-section notes capped at ten plus “and N more”; +- both ambiguous template rows result in explicit `Refuse`; +- real crawl evidence can infer `section_segment = 1`; +- refused slots do not appear in rendered output and their reasons appear in notes. + +- [ ] **Step 2: Run narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::evidence +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::gpt_slots +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::page_patterns +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::crawl_plan +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::unit_template +``` + +Expected: each new regression reproduces its review finding. + +- [ ] **Step 3: Implement evidence-preserving discovery** + +Use the last matching `char_indices` byte boundary for shared prefixes. Remove an empty-page marker whenever a later profile yields slots. Require `(useful shared prefix || group size >= 3)` before classifying disjoint same-shape slots as fragments; emit an ambiguity diagnostic otherwise. + +Group normalized collisions within a page before deduplication. When a group has multiple raw div IDs, keep raw entries, make their generated IDs unique, and attach a collision note. Restrict ephemeral hex matching to tokens containing at least one `a..f`, or an explicit UUID shape; never treat all-digit identifiers as hashes. Reject gampad fallback when parsed `dids` contains a comma. + +- [ ] **Step 4: Implement conservative patterns/templates** + +Emit the observed short path for locale landing pages, escape literal prefixes, decode only for filtering while retaining encoded request paths for matching, and cap notes. Teach crawl planning to carry/infer the section depth used by page-pattern generation. + +Delete the tautological witness check and move its explanatory invariant into `analyse_slot` docs. Keep the existing conservative `Refuse` result for non-derivable slugs and unwitnessed roots. Filter all `Refuse` decisions before `RenderSlot` creation and push each reason into notes. + +- [ ] **Step 5: Verify GREEN** + +Run all five narrow commands again, then: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate +``` + +Expected: the generate module suite passes. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate +git commit -m "Preserve ad template crawl evidence" +``` + +## Task 7: Make slot persistence and dry-run output safe + +**Files:** +- Modify: `Cargo.toml` +- Modify: `Cargo.lock` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/validate.rs` + +- [ ] **Step 1: Add failing persistence tests** + +Cover: + +- trailing comments after the final slot; +- a multiline string line beginning `[foo]`; +- an array continuation beginning `[300, 250]`; +- non-contiguous slot tables; +- byte-identical unrelated sections/comments and CRLF preservation; +- end-to-end `--replace` through `run_update_slots`; +- dry-run source file byte identity; +- stdout contains only a zero-context unified diff of managed + creative-opportunities changes and does not contain `admin_password` or + unrelated config; +- notes/rollback warning go to stderr; +- a concurrent source edit between initial read and write is refused; +- rerun unions formats and reports broad-prefix collapse. + +Change `run_update_slots` to accept separate writers: + +```rust +pub(crate) fn run_update_slots( + request: &UpdateSlotsRequest<'_>, + collectors: &[(&str, &dyn AuditCollector)], + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()>; +``` + +- [ ] **Step 2: Run persistence tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::slot_toml +cargo test --package trusted-server-cli --target "$HOST_TARGET" update_slots +``` + +Expected: current line scanner corrupts/preserves incorrectly and dry-run leaks the complete config. + +- [ ] **Step 3: Implement a TOML-aware managed edit** + +Parse the source as `DocumentMut` and update the complete managed field set: +`creative_opportunities.slot`, `gam_network_id`, `section_root`, and +`section_segment`. Insert the generated array-of-tables and upsert only scalar +values that generation actually inferred. A generated `None` preserves the +existing scalar on both merge and `--replace`; absence of fresh evidence is +never an instruction to delete operator configuration. Retain decorations on +all other items. Before returning, parse both documents and compare canonical +clones with all four managed fields removed; return an error if any other item +differs. Preserve CRLF after serialization. Add regression cases in Step 1 for +an unresolved network ID and literal-only rerun retaining existing +`gam_network_id`/section policy. + +Document `splice_creative_slots` at its definition and remove the orphaned comments. Replace the `let _ = network_id` presence check with `keys.network_id.is_none()` logic. + +- [ ] **Step 4: Implement secret-safe dry-run and stale-read protection** + +Add `similar` as a workspace/CLI dependency and render a zero-context unified +diff between the old and new managed creative-opportunities projection. The +projection contains only `gam_network_id`, `section_root`, `section_segment`, +and the slot array, so every generated scalar change is visible without +including unrelated operator keys: + +```rust +let diff = similar::TextDiff::from_lines(old_managed, new_managed); +writeln!(out, "{}", diff.unified_diff().context_radius(0).header("configured creative opportunities", "generated creative opportunities"))?; +``` + +Send all notes to `err`. Immediately before atomic rename, re-read the config and compare it with the original bytes; refuse on mismatch. Do not perform this check on dry-run because no write occurs. + +In `merge_render_slots`, union discovered formats into a matching existing slot and count how many discovered slots map to each existing prefix; report counts greater than one. + +- [ ] **Step 5: Verify GREEN** + +Run both narrow commands again and confirm all tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add Cargo.toml Cargo.lock crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/commands/audit/generate +git commit -m "Preserve operator config during slot generation" +``` + +## Task 8: Complete documentation, test hygiene, and CI enforcement + +**Files:** +- Modify: `docs/guide/cli.md` +- Modify: `scripts/test-cli.sh` +- Modify: `.github/workflows/test.yml` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: touched Rust tests and comments under `crates/trusted-server-cli/src/` + +- [ ] **Step 1: Add/restore parser and CI guard tests** + +Restore the `audit` no-`--adapter` parser test. Add a script contract that sets `TS_AUDIT_BROWSER_TESTS=1`; browser fixture tests panic when that variable is set and Chrome cannot be resolved. Configure the workflow with a browser setup action or the runner's installed Chrome path and export `CHROME` before `scripts/test-cli.sh`. + +- [ ] **Step 2: Replace sensitive-looking fixtures and stale assertions** + +Replace `88059007`, `autoblog`, `car-research`/`carresearch`, and distinctive div tokens introduced by this PR with `123456789`, `publisher`, `/site-news`/`sitenews`, and neutral `ex_...` values. Update comments to describe shapes rather than customers. + +Correct all touched `expect` messages to start with `should`, remove redundant crate/file `dead_code` allowances and annotate only genuinely deferred fields, reorder `Audit`, simplify the Prebid query parser so keys—not substrings—are matched, and bind legacy URLs directly without an impossible `expect`. + +- [ ] **Step 3: Document the complete operator contract** + +In `docs/guide/cli.md`, document: + +- `config ad-templates lint|match|check|explain` and every flag; +- shared `--app-config`, `--manifest`, and `--no-env` behavior; +- `audit ad-templates generate|verify` browser/profile/proxy/consent/settle flags; +- dry-run stdout diff versus stderr notes; +- exit 0 success, exit 1 assertion drift, exit 2 tool/configuration error; +- refused slots are omitted with reasons; +- locale-prefixed inference and section depth; +- `Unconfirmable` strict behavior and optional evidence phase. + +Update the existing design/output examples where the wire contract changed. + +- [ ] **Step 4: Run format and focused checks** + +Run: + +```bash +cargo fmt --all -- --check +cd docs && npm run format +``` + +Expected: both commands exit 0. + +- [ ] **Step 5: Commit** + +```bash +git add .github/workflows/test.yml scripts/test-cli.sh docs crates/trusted-server-cli/src +git commit -m "Document and enforce ad template audit contracts" +``` + +## Task 9: Run full verification and repair regressions + +**Files:** +- Modify only files implicated by a failing check. + +- [ ] **Step 1: Run format and CLI/browser tests** + +```bash +cargo fmt --all -- --check +./scripts/test-cli.sh +``` + +Expected: exit 0; browser fixture output shows tests executed rather than skipped. + +- [ ] **Step 2: Run repository target suites** + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: all suites exit 0. + +- [ ] **Step 3: Run all target-matched clippy gates** + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings +``` + +Expected: all commands exit 0 with no warnings. + +- [ ] **Step 4: Run cross-adapter parity gates** + +```bash +cargo fmt --manifest-path crates/trusted-server-integration-tests/Cargo.toml -- --check +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +cargo clippy --manifest-path crates/trusted-server-integration-tests/Cargo.toml --all-targets -- -D warnings +``` + +Expected: formatting, parity tests, and integration-test clippy exit 0. + +- [ ] **Step 5: Run JavaScript and documentation checks** + +```bash +cd crates/trusted-server-js/lib && npx vitest run && npm run format && node build-all.mjs +cd ../../.. && cd docs && npm run format +``` + +Expected: tests/build/format exit 0. + +- [ ] **Step 6: Inspect the final diff against the review** + +Run: + +```bash +git diff --check origin/main...HEAD +git status --short +``` + +Walk the 28-thread traceability table and every summary category in the design spec. Confirm each has a code/doc/test resolution or an evidence-backed response. + +- [ ] **Step 7: Commit any verification-only corrections** + +If verification required changes, inspect `git diff --name-only`, stage each +listed path explicitly (never `git add .`), and commit them as `Resolve ad +template review regressions`. Record those exact paths in the execution log. +Skip this commit when verification required no changes. + +## Task 10: Publish and answer GitHub review threads + +**Files:** +- No repository files unless publication reveals a conflict. + +- [ ] **Step 1: Push the verified branch** + +```bash +git push origin feature/ts-cli-ad-templates +``` + +Expected: push succeeds and PR 823 shows the verified head commit. + +- [ ] **Step 2: Correct the PR description** + +Change the legacy alias statement to say bare `ts audit ` aliases to `ts audit generate `. Preserve all unrelated PR-body content. + +- [ ] **Step 3: Reply to every inline thread** + +For each ID in the spec traceability table, post through: + +```bash +gh api repos/IABTechLab/trusted-server/pulls/823/comments//replies -f body='' +``` + +Each reply must name the concrete behavior changed and, where useful, the focused test. For question threads, state the chosen behavior: union formats and diagnose broad prefixes; default consent assumption on; keep conservative refusal and align docs; allow only same-host HTTP-to-HTTPS upgrades; remove the unsupported EdgeZero model. + +- [ ] **Step 4: Verify publication** + +Query PR 823's head SHA, review comments, checks, and unresolved threads. Confirm all 28 inline comments have one reply and no reply claims a fix absent from the pushed diff. + +- [ ] **Step 5: Report the result** + +Summarize commits, verification commands, any environment limitation, PR link, and thread reply count. Do not claim checks pass without fresh output from Task 9. From bca89ef23fedba842050720f0ffd20b0b17e69b0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 14:12:38 +0530 Subject: [PATCH 314/395] Align ad stack gate diagnostics with runtime --- .../src/commands/audit/generate/mod.rs | 4 +- .../commands/audit/generate/page_patterns.rs | 2 +- .../src/creative_opportunities.rs | 156 +++++++++++++----- crates/trusted-server-core/src/publisher.rs | 89 ++-------- 4 files changed, 129 insertions(+), 122 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 454abb16d..a6c7eb12f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -16,7 +16,7 @@ use std::path::{Path, PathBuf}; use serde::Serialize; use trusted_server_core::creative_opportunities::{ - CreativeOpportunitiesConfig, compile_page_pattern, + CreativeOpportunitiesConfig, validate_page_pattern, }; use url::Url; @@ -885,7 +885,7 @@ fn build_render_slots( fn validate_page_patterns(patterns: &[String]) -> CliResult<()> { let invalid: Vec = patterns .iter() - .filter_map(|pattern| compile_page_pattern(pattern).err()) + .filter_map(|pattern| validate_page_pattern(pattern).err()) .collect(); if invalid.is_empty() { return Ok(()); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs index 5c8c3fd27..c779904b1 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs @@ -118,7 +118,7 @@ mod tests { let patterns = patterns_for_paths(["/", "/news/story", "/car-research/x"], 0); for pattern in &patterns { - trusted_server_core::creative_opportunities::compile_page_pattern(pattern) + trusted_server_core::creative_opportunities::validate_page_pattern(pattern) .unwrap_or_else(|error| { panic!("emitted pattern `{pattern}` must compile: {error}") }); diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index c90c8bd63..4c3a5986a 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -843,20 +843,34 @@ pub struct PrebidSlotParams { /// Returns an error string when the pattern compiles neither directly nor after /// normalisation. /// +pub(crate) fn compile_page_pattern(pattern: &str) -> Result { + Pattern::new(pattern) + .or_else(|_| Pattern::new(&pattern.replace("**", "*"))) + .map_err(|error| format!("page pattern '{pattern}' is not a valid glob: {error}")) +} + +/// Validates a [`page_patterns`](CreativeOpportunitySlot::page_patterns) entry +/// using the runtime's normalisation. +/// +/// This exposes validation without leaking the runtime's `glob::Pattern` type +/// into the public API. +/// +/// # Errors +/// +/// Returns an error string when the pattern compiles neither directly nor after +/// the runtime's `**` to `*` normalisation. +/// /// # Examples /// /// ``` -/// use trusted_server_core::creative_opportunities::compile_page_pattern; +/// use trusted_server_core::creative_opportunities::validate_page_pattern; /// -/// assert!(compile_page_pattern("/news/*").is_ok()); -/// // `**` in a position the glob crate rejects is normalised to `*`. -/// assert!(compile_page_pattern("/20**").is_ok()); -/// assert!(compile_page_pattern("[").is_err()); +/// assert!(validate_page_pattern("/news/*").is_ok()); +/// assert!(validate_page_pattern("/20**").is_ok()); +/// assert!(validate_page_pattern("[").is_err()); /// ``` -pub fn compile_page_pattern(pattern: &str) -> Result { - Pattern::new(pattern) - .or_else(|_| Pattern::new(&pattern.replace("**", "*"))) - .map_err(|error| format!("page pattern '{pattern}' is not a valid glob: {error}")) +pub fn validate_page_pattern(pattern: &str) -> Result<(), String> { + compile_page_pattern(pattern).map(|_| ()) } /// Validates that a slot ID contains only safe characters. @@ -926,11 +940,35 @@ pub enum AdStackGateName { AuctionEnabled, } +impl AdStackGateName { + const ALL: [Self; 7] = [ + Self::MethodGet, + Self::Navigation, + Self::NotPrefetch, + Self::NotBot, + Self::MatchedSlots, + Self::ConsentAllowsAuction, + Self::AuctionEnabled, + ]; + + fn blocks(self, input: AdStackGateInput) -> bool { + match self { + Self::MethodGet => !input.method_get, + Self::Navigation => !input.navigation, + Self::NotPrefetch => input.prefetch, + Self::NotBot => input.bot, + Self::MatchedSlots => !input.matched_slots, + Self::ConsentAllowsAuction => input.consent_allows_auction == Some(false), + Self::AuctionEnabled => !input.auction_enabled, + } + } +} + /// Inputs to [`evaluate_ad_stack_gate`]. /// /// `consent_allows_auction` is tri-state: `Some(true)` allows, `Some(false)` /// blocks, and `None` means the caller cannot prove the consent state. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Eq, PartialEq)] pub struct AdStackGateInput { /// Request method is `GET`. pub method_get: bool, @@ -942,7 +980,12 @@ pub struct AdStackGateInput { pub bot: bool, /// At least one configured slot matches the request path. pub matched_slots: bool, - /// Whether consent allows the auction; `None` when unprovable. + /// Whether consent allows the auction. + /// + /// `Some(true)` allows the auction, `Some(false)` blocks it, and `None` + /// means the caller cannot prove either state. Unknown consent is not a + /// denial: it produces [`RuntimeAdStackExpected::Unknown`] when every known + /// boolean gate passes. pub consent_allows_auction: Option, /// The global `[auction].enabled` kill switch. pub auction_enabled: bool, @@ -955,14 +998,16 @@ pub struct AdStackGateInput { pub struct AdStackGateResult { /// The three-state ad-stack expectation. pub expected: RuntimeAdStackExpected, - blocking_gates: Vec, + input: AdStackGateInput, } impl AdStackGateResult { /// Returns the gates that blocked the server-side ad stack. #[must_use] - pub fn blocking_gates(&self) -> &[AdStackGateName] { - &self.blocking_gates + pub fn blocking_gates(&self) -> impl Iterator + '_ { + AdStackGateName::ALL + .into_iter() + .filter(|gate| gate.blocks(self.input)) } } @@ -979,30 +1024,14 @@ impl AdStackGateResult { /// `bot` block when `true`. #[must_use] pub fn evaluate_ad_stack_gate(input: AdStackGateInput) -> AdStackGateResult { - let mut blocking_gates = Vec::new(); - if !input.method_get { - blocking_gates.push(AdStackGateName::MethodGet); - } - if !input.navigation { - blocking_gates.push(AdStackGateName::Navigation); - } - if input.prefetch { - blocking_gates.push(AdStackGateName::NotPrefetch); - } - if input.bot { - blocking_gates.push(AdStackGateName::NotBot); - } - if !input.matched_slots { - blocking_gates.push(AdStackGateName::MatchedSlots); - } - if input.consent_allows_auction == Some(false) { - blocking_gates.push(AdStackGateName::ConsentAllowsAuction); - } - if !input.auction_enabled { - blocking_gates.push(AdStackGateName::AuctionEnabled); - } - - let expected = if !blocking_gates.is_empty() { + let known_gate_blocks = !input.method_get + || !input.navigation + || input.prefetch + || input.bot + || !input.matched_slots + || input.consent_allows_auction == Some(false) + || !input.auction_enabled; + let expected = if known_gate_blocks { RuntimeAdStackExpected::No } else if input.consent_allows_auction.is_none() { RuntimeAdStackExpected::Unknown @@ -1010,10 +1039,7 @@ pub fn evaluate_ad_stack_gate(input: AdStackGateInput) -> AdStackGateResult { RuntimeAdStackExpected::Yes }; - AdStackGateResult { - expected, - blocking_gates, - } + AdStackGateResult { expected, input } } #[cfg(test)] @@ -1033,7 +1059,7 @@ mod tests { }); assert_eq!(result.expected, RuntimeAdStackExpected::Yes); - assert!(result.blocking_gates().is_empty()); + assert_eq!(result.blocking_gates().count(), 0); } #[test] @@ -1052,7 +1078,7 @@ mod tests { assert!( result .blocking_gates() - .contains(&AdStackGateName::AuctionEnabled) + .any(|gate| gate == AdStackGateName::AuctionEnabled) ); } @@ -1098,6 +1124,48 @@ mod tests { } } + #[test] + fn ad_stack_gate_with_unknown_consent_matches_known_boolean_gates() { + for bits in 0u8..64 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: None, + auction_enabled: bits & 32 != 0, + }; + let known_gates_pass = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.auction_enabled; + let expected = if known_gates_pass { + RuntimeAdStackExpected::Unknown + } else { + RuntimeAdStackExpected::No + }; + + assert_eq!( + evaluate_ad_stack_gate(input).expected, + expected, + "should match unknown-consent gate semantics for bits={bits}" + ); + } + } + + #[test] + fn validate_page_pattern_preserves_specific_compile_error() { + let error = validate_page_pattern("[").expect_err("should reject invalid glob"); + + assert!( + error.contains("page pattern '[' is not a valid glob"), + "should retain the invalid pattern in the error: {error}" + ); + } + fn make_slot(id: &str, patterns: Vec<&str>) -> CreativeOpportunitySlot { CreativeOpportunitySlot { id: id.to_string(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 9ac3b67b9..2d3fa4455 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -51,6 +51,9 @@ use crate::auction::types::{ use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; +use crate::creative_opportunities::{ + AdStackGateInput, RuntimeAdStackExpected, evaluate_ad_stack_gate, +}; use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; @@ -1800,35 +1803,6 @@ pub(crate) fn is_prefetch_request(req: &Request) -> bool { header("sec-purpose") || header("purpose") } -/// Returns true only when the publisher request should run the full -/// server-side ad stack: auction dispatch plus initial ad-slot injection. -/// -/// `auction_enabled` is the global `[auction].enabled` kill switch — when -/// false, no automatic server-side auction or ad-slot injection runs. -pub(crate) fn should_run_server_side_ad_stack( - is_get: bool, - is_navigation: bool, - is_prefetch: bool, - is_bot: bool, - has_matched_slots: bool, - consent_allows_auction: bool, - auction_enabled: bool, -) -> bool { - crate::creative_opportunities::evaluate_ad_stack_gate( - crate::creative_opportunities::AdStackGateInput { - method_get: is_get, - navigation: is_navigation, - prefetch: is_prefetch, - bot: is_bot, - matched_slots: has_matched_slots, - consent_allows_auction: Some(consent_allows_auction), - auction_enabled, - }, - ) - .expected - == crate::creative_opportunities::RuntimeAdStackExpected::Yes -} - /// Write winning bids from an auction result into the shared `ad_bids_state` lock. /// Build the request origin (`scheme://host`, where `host` includes any port) /// used to emit absolute first-party URLs in inline creatives. Returns an empty @@ -2719,15 +2693,17 @@ pub async fn handle_publisher_request( // (storage/access) before firing. Known non-GDPR jurisdictions are free. let consent_allows_auction = consent_allows_server_side_auction(&consent_context); - let should_run_ad_stack = should_run_server_side_ad_stack( - is_get, - is_navigation, - is_prefetch, - is_bot, - !matched_slots.is_empty(), - consent_allows_auction, - auction.orchestrator.is_enabled(), - ); + let should_run_ad_stack = evaluate_ad_stack_gate(AdStackGateInput { + method_get: is_get, + navigation: is_navigation, + prefetch: is_prefetch, + bot: is_bot, + matched_slots: !matched_slots.is_empty(), + consent_allows_auction: Some(consent_allows_auction), + auction_enabled: auction.orchestrator.is_enabled(), + }) + .expected + == RuntimeAdStackExpected::Yes; let should_run_auction = should_run_ad_stack; // Diagnostic: shows which gate suppresses the server-side auction. Pair with // the `EC context: ... jurisdiction=...` line from EC-context construction @@ -5988,43 +5964,6 @@ mod tests { ); } - #[test] - fn server_side_ad_stack_runs_only_when_all_auction_gates_pass() { - assert!( - should_run_server_side_ad_stack(true, true, false, false, true, true, true), - "GET, real navigation, matched slots, and consent should run TS ad stack" - ); - - assert!( - !should_run_server_side_ad_stack(false, true, false, false, true, true, true), - "non-GET requests should skip TS ad stack" - ); - assert!( - !should_run_server_side_ad_stack(true, false, false, false, true, true, true), - "non-document requests should skip TS ad stack" - ); - assert!( - !should_run_server_side_ad_stack(true, true, true, false, true, true, true), - "prefetch requests should skip TS ad stack and injection" - ); - assert!( - !should_run_server_side_ad_stack(true, true, false, true, true, true, true), - "bot requests should skip TS ad stack and injection" - ); - assert!( - !should_run_server_side_ad_stack(true, true, false, false, false, true, true), - "requests with no matching slots should skip TS ad stack" - ); - assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, false, true), - "requests without required consent should skip TS ad stack and injection" - ); - assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, true, false), - "disabled [auction].enabled kill switch should skip TS ad stack and injection" - ); - } - #[tokio::test] async fn body_close_hold_loop_processes_close_tail_before_reading_post_body_chunks() { let settings = create_test_settings(); From 1b418cca2f944f0112647a05852ebd48bf575213 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 14:16:48 +0530 Subject: [PATCH 315/395] Match ad template verification to runtime behavior --- .../src/ad_templates/compare.rs | 83 +++++----- .../src/ad_templates/expected.rs | 65 ++++---- .../src/ad_templates/output.rs | 44 +++++- .../src/commands/audit/ad_templates.rs | 142 +++++++++++++++++- 4 files changed, 251 insertions(+), 83 deletions(-) diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs index 215a6a300..0eb736209 100644 --- a/crates/trusted-server-cli/src/ad_templates/compare.rs +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -14,6 +14,7 @@ use serde::Deserialize; +use trusted_server_core::auction::types::MediaType; use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; use crate::ad_templates::expected::ExpectedSlot; @@ -126,6 +127,8 @@ pub enum SlotStatus { Partial, /// No DOM or GPT evidence confirms the slot. Missing, + /// The checker cannot confirm this slot type; this is not page drift. + Unconfirmable, } /// The verification result for one audited page. @@ -164,7 +167,7 @@ pub struct SlotResult { /// The confirmation status. pub status: SlotStatus, /// The phase the confirming evidence was observed in. - pub phase: EvidencePhase, + pub phase: Option, /// The live evidence observed for this slot. pub evidence: SlotEvidence, /// Slot-level warnings (size, provider, etc.). @@ -233,7 +236,7 @@ fn banner_sizes(expected: &ExpectedSlot) -> Vec<(u32, u32)> { expected .formats .iter() - .filter(|format| format.media_type == "banner") + .filter(|format| format.media_type == MediaType::Banner) .map(|format| (format.width, format.height)) .collect() } @@ -267,7 +270,7 @@ pub fn compare_page_evidence( "gam_unit_path_unrenderable", format!( "slot `{}` gam_unit_path template renders past GAM's unit-path byte limit \ - for this page's section; the runtime rejects this config", + for this page's section; the runtime omits this slot on this path", slot.id ), )); @@ -285,7 +288,12 @@ pub fn compare_page_evidence( slot.id ), )); - (SlotStatus::Partial, dom_id, Some(gpt.clone()), gpt.phase) + ( + SlotStatus::Unconfirmable, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) } else if gpt.sizes.is_empty() { warnings.push(warning( "out_of_page_slot", @@ -294,7 +302,12 @@ pub fn compare_page_evidence( slot.id ), )); - (SlotStatus::Partial, dom_id, Some(gpt.clone()), gpt.phase) + ( + SlotStatus::Unconfirmable, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) } else if banner.iter().any(|size| gpt.sizes.contains(size)) { let extra: Vec<(u32, u32)> = gpt .sizes @@ -322,7 +335,12 @@ pub fn compare_page_evidence( ), )); } - (SlotStatus::Confirmed, dom_id, Some(gpt.clone()), gpt.phase) + ( + SlotStatus::Confirmed, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) } else { warnings.push(warning( "incompatible_sizes", @@ -331,7 +349,12 @@ pub fn compare_page_evidence( slot.id ), )); - (SlotStatus::Partial, dom_id, Some(gpt.clone()), gpt.phase) + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) } } else if let Some(dom) = resolved { warnings.push(warning( @@ -342,25 +365,12 @@ pub fn compare_page_evidence( SlotStatus::Partial, Some(dom.dom_id.clone()), None, - dom.phase, + Some(dom.phase), ) } else { - (SlotStatus::Missing, None, None, EvidencePhase::InitialLoad) + (SlotStatus::Missing, None, None, None) }; - if let Some(aps_slot_id) = &slot.aps_slot_id { - let matched = evidence - .aps_calls - .iter() - .any(|call| &call.slot_id == aps_slot_id); - if !matched { - warnings.push(warning( - "aps_evidence_missing", - format!("configured APS slot `{aps_slot_id}` had no fetchBids evidence"), - )); - } - } - slots.push(SlotResult { id: slot.id.clone(), status, @@ -454,11 +464,10 @@ mod tests { .map(|&(width, height)| ExpectedFormat { width, height, - media_type: "banner".to_string(), + media_type: MediaType::Banner, }) .collect(), providers: providers.iter().copied().map(String::from).collect(), - aps_slot_id: providers.contains(&"aps").then(|| id.to_string()), page_patterns: Vec::new(), } } @@ -471,10 +480,9 @@ mod tests { formats: vec![ExpectedFormat { width: 0, height: 0, - media_type: "video".to_string(), + media_type: MediaType::Video, }], providers: Vec::new(), - aps_slot_id: None, page_patterns: Vec::new(), } } @@ -669,7 +677,7 @@ mod tests { } #[test] - fn non_banner_only_slot_is_partial() { + fn non_banner_only_slot_is_unconfirmable_and_does_not_fail_strict() { let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); let evidence = evidence( vec![dom("ad-video-0")], @@ -683,13 +691,17 @@ mod tests { RuntimeGateSummary::unknown_allowed(), ); - assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert_eq!(result.slots[0].status, SlotStatus::Unconfirmable); assert!( result.slots[0] .warnings .iter() .any(|w| w.code == "unsupported_format") ); + assert!( + !result.strict_failed(), + "checker limitations should not fail strict" + ); } #[test] @@ -739,13 +751,17 @@ mod tests { RuntimeGateSummary::unknown_allowed(), ); - assert_ne!(result.slots[0].status, SlotStatus::Confirmed); + assert_eq!(result.slots[0].status, SlotStatus::Unconfirmable); assert!( result.slots[0] .warnings .iter() .any(|w| w.code == "out_of_page_slot") ); + assert!( + !result.strict_failed(), + "out-of-page slots are not confirmable by this checker" + ); } #[test] @@ -774,7 +790,7 @@ mod tests { } #[test] - fn aps_missing_warns_but_keeps_confirmed() { + fn server_side_aps_config_does_not_require_client_fetch_bids_evidence() { let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); let evidence = evidence( vec![dom("ad-atf-0")], @@ -793,12 +809,7 @@ mod tests { SlotStatus::Confirmed, "missing APS does not flip status" ); - assert!( - result.slots[0] - .warnings - .iter() - .any(|w| w.code == "aps_evidence_missing") - ); + assert!(result.slots[0].warnings.is_empty()); assert!( !result.strict_failed(), "provider warning alone must not fail strict" diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs index 549ec0a57..e24488375 100644 --- a/crates/trusted-server-cli/src/ad_templates/expected.rs +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -27,18 +27,13 @@ pub struct ExpectedSlot { /// Resolved GAM unit path: the rendered `gam_unit_path` template (or /// `//` when the slot has none). /// - /// `None` when a dynamic template renders beyond GAM's unit-path byte limit - /// for this path's section. Runtime validation rejects such a config, so - /// this only occurs for a config that would fail to load; the slot is then - /// reported unconfirmable rather than matched against a wrong path. + /// `None` only for manually constructed comparison fixtures. Projection + /// omits a slot when the runtime cannot render it for this path. pub gam_unit_path: Option, /// Configured ad formats. pub formats: Vec, /// Configured provider names, in `aps`, `prebid` order. pub providers: Vec, - /// Configured APS slot ID, when the `aps` provider is set. Used to match - /// `apstag.fetchBids` evidence; not part of the §8 JSON output. - pub aps_slot_id: Option, /// Glob patterns configured for this slot. pub page_patterns: Vec, } @@ -50,8 +45,8 @@ pub struct ExpectedFormat { pub width: u32, /// Creative height in pixels. pub height: u32, - /// Media type rendered as a stable string (`banner`, `video`, `native`). - pub media_type: String, + /// Configured media type. + pub media_type: MediaType, } /// Projects the slots matching `path` into stable expected-slot records. @@ -71,22 +66,24 @@ pub fn expected_slots_for_path(path: &str, config: &CreativeOpportunitiesConfig) let section = config.section_for_path(path); let slots = match_slots(&config.slot, path) .into_iter() - .map(|slot| ExpectedSlot { - id: slot.id.clone(), - div_id: slot.resolved_div_id().to_string(), - gam_unit_path: slot.render_gam_unit_path(&config.gam_network_id, §ion), - formats: slot - .formats - .iter() - .map(|format| ExpectedFormat { - width: format.width, - height: format.height, - media_type: media_type_str(&format.media_type).to_string(), - }) - .collect(), - providers: provider_names(slot), - aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), - page_patterns: slot.page_patterns.clone(), + .filter_map(|slot| { + let gam_unit_path = slot.render_gam_unit_path(&config.gam_network_id, §ion)?; + Some(ExpectedSlot { + id: slot.id.clone(), + div_id: slot.resolved_div_id().to_string(), + gam_unit_path: Some(gam_unit_path), + formats: slot + .formats + .iter() + .map(|format| ExpectedFormat { + width: format.width, + height: format.height, + media_type: format.media_type.clone(), + }) + .collect(), + providers: provider_names(slot), + page_patterns: slot.page_patterns.clone(), + }) }) .collect(); @@ -96,14 +93,6 @@ pub fn expected_slots_for_path(path: &str, config: &CreativeOpportunitiesConfig) } } -fn media_type_str(media_type: &MediaType) -> &'static str { - match media_type { - MediaType::Banner => "banner", - MediaType::Video => "video", - MediaType::Native => "native", - } -} - fn provider_names( slot: &trusted_server_core::creative_opportunities::CreativeOpportunitySlot, ) -> Vec { @@ -204,7 +193,7 @@ mod tests { vec![ExpectedFormat { width: 300, height: 250, - media_type: "banner".to_string(), + media_type: MediaType::Banner, }] ); } @@ -263,7 +252,7 @@ mod tests { } #[test] - fn expected_slots_report_unrenderable_dynamic_template_as_none() { + fn expected_slots_omit_dynamic_template_the_runtime_cannot_render() { // A `{section}` template that renders past GAM's 100-byte unit-path // limit. `validate_runtime` rejects this config, so the verifier reports // the slot as unconfirmable rather than matching a truncated path. @@ -282,9 +271,9 @@ mod tests { let long_path = format!("/{}", "a".repeat(60)); let expected = expected_slots_for_path(&long_path, &config); - assert_eq!( - expected.slots[0].gam_unit_path, None, - "an over-limit dynamic render should project as None" + assert!( + expected.slots.is_empty(), + "the runtime omits an over-limit dynamic slot on this path" ); } diff --git a/crates/trusted-server-cli/src/ad_templates/output.rs b/crates/trusted-server-cli/src/ad_templates/output.rs index 9a1190652..0df64d52f 100644 --- a/crates/trusted-server-cli/src/ad_templates/output.rs +++ b/crates/trusted-server-cli/src/ad_templates/output.rs @@ -49,7 +49,10 @@ pub fn escape_terminal_text(value: &str) -> Cow<'_, str> { /// Whether `ch` can act as a terminal control code (C0, DEL, or C1). fn is_terminal_control(ch: char) -> bool { let code = ch as u32; - code < 0x20 || (0x7f..=0x9f).contains(&code) + code < 0x20 + || (0x7f..=0x9f).contains(&code) + || (0x202a..=0x202e).contains(&code) + || (0x2066..=0x2069).contains(&code) } /// Confirmation status for a single configured slot. @@ -62,6 +65,8 @@ pub enum SlotStatus { Partial, /// No DOM or GPT evidence confirms the slot. Missing, + /// The checker does not support confirming this slot type. + Unconfirmable, } /// JSON rendering of the runtime ad-stack expectation. @@ -195,7 +200,8 @@ pub struct SlotJson { /// The slot's confirmation status. pub status: SlotStatus, /// The phase the confirming evidence was observed in. - pub phase: EvidencePhaseJson, + #[serde(skip_serializing_if = "Option::is_none")] + pub phase: Option, /// The configured shape of the slot (no `id`/`page_patterns` per §8). pub configured: ConfiguredJson, /// The live evidence observed for this slot. @@ -292,7 +298,7 @@ impl VerificationReport { slots: vec![SlotJson { id: "atf".to_string(), status: SlotStatus::Confirmed, - phase: EvidencePhaseJson::InitialLoad, + phase: Some(EvidencePhaseJson::InitialLoad), configured: ConfiguredJson { div_id: "ad-atf-".to_string(), gam_unit_path: Some("/123/news/atf".to_string()), @@ -393,6 +399,11 @@ mod tests { "del\\u{007F}c1\\u{009B}", "DEL and the C1 range should be escaped too" ); + assert_eq!( + escape_terminal_text("safe\u{202E}forged\u{2066}tail"), + "safe\\u{202E}forged\\u{2066}tail", + "Unicode bidi controls should be rendered inert" + ); } #[test] @@ -441,4 +452,31 @@ mod tests { ); assert_eq!(value["ok"], false); } + + #[test] + fn missing_slot_json_omits_evidence_phase() { + let slot = SlotJson { + id: "missing".to_string(), + status: SlotStatus::Missing, + phase: None, + configured: ConfiguredJson { + div_id: "ad-missing-".to_string(), + gam_unit_path: Some("/123/publisher/missing".to_string()), + formats: Vec::new(), + providers: Vec::new(), + }, + evidence: SlotEvidenceJson { + dom_id: None, + gpt: None, + }, + warnings: Vec::new(), + }; + + let value = serde_json::to_value(slot).expect("should serialize missing slot"); + + assert!( + value.get("phase").is_none(), + "missing evidence should not claim an initial-load phase" + ); + } } diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs index 8250e1cde..56d53921a 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -8,6 +8,7 @@ use std::io::{self, Write}; +use trusted_server_core::auction::types::MediaType; use trusted_server_core::creative_opportunities::{ AdStackGateInput, CreativeOpportunitiesConfig, evaluate_ad_stack_gate, }; @@ -203,6 +204,7 @@ fn build_page( let strict_failed = result.strict_failed(); let mut warnings: Vec = collected.warnings.to_vec(); + warnings.extend(evidence.warnings.iter().cloned()); if requested_path != final_path { warnings.push(Warning { code: "redirected".to_string(), @@ -321,7 +323,7 @@ fn to_slot_json(expected: &ExpectedSlot, result: &SlotResult) -> SlotJson { SlotJson { id: result.id.clone(), status: to_status(result.status), - phase: to_phase(result.phase), + phase: result.phase.map(to_phase), configured: ConfiguredJson { div_id: expected.div_id.clone(), gam_unit_path: expected.gam_unit_path.clone(), @@ -331,7 +333,7 @@ fn to_slot_json(expected: &ExpectedSlot, result: &SlotResult) -> SlotJson { .map(|format| FormatJson { width: format.width, height: format.height, - media_type: format.media_type.clone(), + media_type: media_type_label(&format.media_type).to_string(), }) .collect(), providers: expected.providers.clone(), @@ -368,6 +370,7 @@ fn to_status(status: CompareStatus) -> SlotStatus { CompareStatus::Confirmed => SlotStatus::Confirmed, CompareStatus::Partial => SlotStatus::Partial, CompareStatus::Missing => SlotStatus::Missing, + CompareStatus::Unconfirmable => SlotStatus::Unconfirmable, } } @@ -378,6 +381,14 @@ fn to_phase(phase: EvidencePhase) -> EvidencePhaseJson { } } +fn media_type_label(media_type: &MediaType) -> &'static str { + match media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + } +} + fn write_json(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { let json = serde_json::to_string_pretty(report) .map_err(|error| format!("failed to serialize verification report: {error}"))?; @@ -398,8 +409,11 @@ fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), S .map_err(write_err) }; + for warning in &report.warnings { + write_warning(out, "", warning)?; + } for page in &report.pages { - writeln!(out, "url: {}", page.url).map_err(write_err)?; + writeln!(out, "url: {}", escape_terminal_text(&page.url)).map_err(write_err)?; if let Some(error) = &page.error { writeln!( out, @@ -411,15 +425,41 @@ fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), S continue; } if let Some(path) = &page.path { - writeln!(out, " path: {path}").map_err(write_err)?; + writeln!(out, " path: {}", escape_terminal_text(path)).map_err(write_err)?; + } + if let Some(expected) = page.runtime_ad_stack_expected { + writeln!(out, " runtime ad stack: {}", runtime_label(expected)).map_err(write_err)?; + } + if let Some(count) = page.matched_slot_count { + writeln!(out, " matched slots: {count}").map_err(write_err)?; + } + if let Some(gates) = &page.gates { + writeln!(out, " gates: {}", gates_label(gates)).map_err(write_err)?; } for slot in &page.slots { - writeln!(out, " slot {}: {}", slot.id, status_label(slot.status)) - .map_err(write_err)?; + writeln!( + out, + " slot {}: {}", + escape_terminal_text(&slot.id), + status_label(slot.status) + ) + .map_err(write_err)?; for warning in &slot.warnings { write_warning(out, " ", warning)?; } } + for extra in &page.extra_evidence { + writeln!( + out, + " extra {} evidence: div={} gam={} sizes={:?} ({})", + escape_terminal_text(&extra.kind), + escape_terminal_text(extra.dom_id.as_deref().unwrap_or("-")), + escape_terminal_text(extra.gam_unit_path.as_deref().unwrap_or("-")), + extra.sizes, + escape_terminal_text(&extra.reason), + ) + .map_err(write_err)?; + } for warning in &page.warnings { write_warning(out, " ", warning)?; } @@ -432,9 +472,39 @@ fn status_label(status: SlotStatus) -> &'static str { SlotStatus::Confirmed => "confirmed", SlotStatus::Partial => "partial", SlotStatus::Missing => "missing", + SlotStatus::Unconfirmable => "unconfirmable", + } +} + +fn runtime_label(expected: RuntimeAdStackExpectedJson) -> &'static str { + match expected { + RuntimeAdStackExpectedJson::Yes => "yes", + RuntimeAdStackExpectedJson::No => "no", + RuntimeAdStackExpectedJson::Unknown => "unknown", } } +fn gate_label(gate: GateState) -> &'static str { + match gate { + GateState::Pass => "pass", + GateState::Fail => "fail", + GateState::Unknown => "unknown", + } +} + +fn gates_label(gates: &Gates) -> String { + format!( + "method_get={} navigation={} not_prefetch={} not_bot={} matched_slots={} auction_enabled={} consent={}", + gate_label(gates.method_get), + gate_label(gates.navigation), + gate_label(gates.not_prefetch), + gate_label(gates.not_bot), + gate_label(gates.matched_slots), + gate_label(gates.auction_enabled), + gate_label(gates.consent_allows_auction), + ) +} + #[allow( clippy::needless_pass_by_value, reason = "used as a map_err fn that receives io::Error by value" @@ -668,6 +738,66 @@ mod tests { assert_eq!(report.pages[0].matched_slot_count, Some(1)); } + #[test] + fn verifier_surfaces_injected_collector_warnings() { + let mut evidence = confirmed_news_evidence(); + evidence.warnings.push(Warning { + code: "fluid_size_ignored".to_string(), + message: "a fluid size could not be compared".to_string(), + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!( + report.pages[0] + .warnings + .iter() + .any(|warning| warning.code == "fluid_size_ignored"), + "collector warning should be visible in the page report" + ); + } + + #[test] + fn human_output_includes_runtime_and_extra_evidence_diagnostics() { + let mut evidence = confirmed_news_evidence(); + evidence.gpt_slots.push(GptSlotEvidence { + gam_unit_path: "/123/publisher/extra".to_string(), + div_id: "ad-extra-0".to_string(), + sizes: vec![(728, 90)], + phase: EvidencePhase::InitialLoad, + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + let mut output = Vec::new(); + + write_human(&mut output, &report).expect("should write human report"); + let output = String::from_utf8(output).expect("should be UTF-8 output"); + + assert!(output.contains("runtime ad stack: unknown")); + assert!(output.contains("matched slots: 1")); + assert!(output.contains("gates: method_get=pass")); + assert!(output.contains("extra gpt evidence")); + } + #[test] fn strict_missing_slot_fails() { let collector = FakeCollector::page( From 1c2dfeb7cd9910faccf0824d1658d0d29f28eb46 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 14:23:10 +0530 Subject: [PATCH 316/395] Define ad template CLI assertion contracts --- Cargo.lock | 1 + crates/trusted-server-cli/Cargo.toml | 1 + .../src/ad_templates/expected.rs | 50 ++-- .../src/commands/audit/ad_templates.rs | 11 +- .../src/commands/audit/mod.rs | 11 +- .../src/commands/config/ad_templates.rs | 221 ++++++++++++------ crates/trusted-server-cli/src/lib.rs | 2 +- crates/trusted-server-cli/src/main.rs | 10 +- crates/trusted-server-cli/src/run.rs | 100 +++++++- 9 files changed, 309 insertions(+), 98 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d429f69d..d2f4f695a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5325,6 +5325,7 @@ dependencies = [ "edgezero-core", "error-stack", "futures", + "http", "http-body-util", "hyper", "hyper-util", diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index eff945b3a..a215bd85c 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -20,6 +20,7 @@ clap = { workspace = true } edgezero-core = { workspace = true } edgezero-cli = { workspace = true } futures = { workspace = true } +http = { workspace = true } log = { workspace = true } regex = { workspace = true } scraper = { workspace = true } diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs index e24488375..373283cb6 100644 --- a/crates/trusted-server-cli/src/ad_templates/expected.rs +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -115,7 +115,14 @@ fn provider_names( /// /// Returns a user-facing string when a `scheme://` input cannot be parsed as a URL. pub fn normalize_path_or_url(input: &str) -> Result { - if input.contains("://") { + let path_input = input.split(['?', '#']).next().unwrap_or(input); + let scheme_prefix = path_input.split_once("://").map(|(scheme, _)| scheme); + let has_url_scheme = scheme_prefix.is_some_and(|scheme| { + let mut chars = scheme.chars(); + chars.next().is_some_and(|ch| ch.is_ascii_alphabetic()) + && chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '+' | '-' | '.')) + }); + if has_url_scheme { let url = Url::parse(input).map_err(|err| format!("invalid URL `{input}`: {err}"))?; let path = url.path(); return Ok(if path.is_empty() { @@ -125,18 +132,13 @@ pub fn normalize_path_or_url(input: &str) -> Result { }); } - let without_fragment = input.split('#').next().unwrap_or(input); - let path = without_fragment - .split('?') - .next() - .unwrap_or(without_fragment); - if path.is_empty() { - Ok("/".to_string()) - } else if path.starts_with('/') { - Ok(path.to_string()) - } else { - Ok(format!("/{path}")) - } + let base = Url::parse("https://path-normalizer.example/") + .expect("should parse static path normalization base"); + let relative = input.trim_start_matches('/'); + let normalized = base + .join(relative) + .map_err(|error| format!("invalid path `{input}`: {error}"))?; + Ok(normalized.path().to_string()) } #[cfg(test)] @@ -298,4 +300,26 @@ mod tests { ); assert_eq!(normalize_path_or_url("").expect("should normalize"), "/"); } + + #[test] + fn normalize_path_or_url_uses_identical_url_rules_for_bare_paths() { + assert_eq!( + normalize_path_or_url("/a/../b").expect("should normalize bare dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("https://example.com/a/../b") + .expect("should normalize URL dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("/a b").expect("should encode bare path"), + "/a%20b" + ); + assert_eq!( + normalize_path_or_url("/r?to=https://example.com") + .expect("query URL should not change input classification"), + "/r" + ); + } } diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs index 56d53921a..7a17d4ac6 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -27,6 +27,7 @@ use crate::commands::audit::AuditAdTemplatesVerifyArgs; use crate::commands::audit::collector::{ AdTemplateCollectorConfig, AuditCollector, BrowserCollectRequest, build_ad_template_init_script, }; +use crate::run::RunOutcome; /// Verifies configured ad-template slots against live page evidence. /// @@ -34,7 +35,7 @@ use crate::commands::audit::collector::{ /// /// Returns a user-facing string when config loading fails, or when verification /// surfaces a page-level error or a `--strict` failure (after writing output). -pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result<(), String> { +pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result { let loaded = crate::app_config::load_settings(&args.config)?; let collector = crate::commands::audit::browser::BrowserCollector::from_opts(&args.browser); let report = build_report( @@ -58,10 +59,12 @@ pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result<(), String write_human(&mut out, &report)?; } - if report.ok { - Ok(()) - } else { + if report.pages.iter().any(|page| page.error.is_some()) { Err("ad-template verification reported problems".to_string()) + } else if report.ok { + Ok(RunOutcome::Success) + } else { + Ok(RunOutcome::AssertionFailed) } } diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index d5787665e..1f7eadd1c 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -16,6 +16,7 @@ use clap::{Args, Subcommand}; use crate::app_config::AppConfigArgs; use crate::commands::audit::collector::BrowserOpts; use crate::commands::audit::page::PageAuditArgs; +use crate::run::RunOutcome; /// Parses and validates an `http`/`https` URL, rejecting all other schemes. /// @@ -52,6 +53,7 @@ pub(crate) fn parse_cookie(raw: &str) -> Result<(String, String), String> { /// `ts audit` arguments: an optional subcommand plus a hidden legacy URL positional. #[derive(Debug, Args)] +#[command(arg_required_else_help = true)] pub(crate) struct AuditArgs { #[command(subcommand)] pub(crate) command: Option, @@ -272,9 +274,11 @@ pub(crate) struct AuditAdTemplatesVerifyArgs { /// /// Returns a user-facing string when no URL or subcommand is provided, or when /// the underlying command fails. -pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { +pub(crate) fn run_audit(args: &AuditArgs) -> Result { match &args.command { - Some(AuditSubcommand::Page(page_args)) => page::run_page(page_args), + Some(AuditSubcommand::Page(page_args)) => { + page::run_page(page_args).map(|()| RunOutcome::Success) + } Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { let loaded = crate::app_config::load_file_settings(&gen_args.config)?; let profiles = gen_args.profiles()?; @@ -315,6 +319,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { &selected, &mut out, ) + .map(|()| RunOutcome::Success) } Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Verify(verify_args))) => { ad_templates::run_verify(verify_args) @@ -324,6 +329,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { let mut out = stdout.lock(); let collector = generate::browser_collector::BrowserAuditCollector::default(); generate::run_generate(generate_args, &collector, &mut out) + .map(|()| RunOutcome::Success) } None => match &args.legacy_url { Some(_) => { @@ -333,6 +339,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { let mut out = stdout.lock(); let collector = generate::browser_collector::BrowserAuditCollector::default(); generate::run_generate(&generate_args, &collector, &mut out) + .map(|()| RunOutcome::Success) } None => Err("provide a URL or a subcommand (`page`, `ad-templates`)".to_string()), }, diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs index 4216db382..14646168a 100644 --- a/crates/trusted-server-cli/src/commands/config/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -2,14 +2,23 @@ use std::collections::BTreeSet; use std::io::{self, Write}; use crate::ad_templates::expected::normalize_path_or_url; +use crate::ad_templates::output::escape_terminal_text; use crate::app_config::{AppConfigArgs, load_settings}; -use clap::{Args, Subcommand}; +use clap::{ArgGroup, Args, Subcommand}; +use http::Method; use trusted_server_core::auction::types::MediaType; use trusted_server_core::creative_opportunities::{ - AdStackGateInput, CreativeOpportunityFormat, CreativeOpportunitySlot, RuntimeAdStackExpected, - evaluate_ad_stack_gate, match_slots, + AdStackGateInput, AdStackGateName, CreativeOpportunityFormat, CreativeOpportunitySlot, + RuntimeAdStackExpected, evaluate_ad_stack_gate, match_slots, validate_page_pattern, }; +use crate::run::RunOutcome; + +enum CheckFailure { + Tool(String), + Assertion(String), +} + #[derive(Debug, Subcommand)] pub enum AdTemplatesCommand { /// Validate ad-template config and summarize deploy-time implications. @@ -40,6 +49,11 @@ pub struct AdTemplatesMatchArgs { } #[derive(Debug, Args)] +#[command(group( + ArgGroup::new("expectation") + .required(true) + .args(["expected_slots", "expect_no_slots"]) +))] pub struct AdTemplatesCheckArgs { #[command(flatten)] pub config: AppConfigArgs, @@ -52,7 +66,7 @@ pub struct AdTemplatesCheckArgs { #[arg(long)] pub expect_no_slots: bool, /// Allow additional matched slots beyond --expected-slot values. - #[arg(long)] + #[arg(long, conflicts_with = "expect_no_slots")] pub allow_extra_slots: bool, } @@ -64,7 +78,7 @@ pub struct AdTemplatesExplainArgs { pub path_or_url: String, /// HTTP method to model. #[arg(long, default_value = "GET")] - pub method: String, + pub method: Method, /// Model a non-navigation request. #[arg(long)] pub non_navigation: bool, @@ -77,9 +91,6 @@ pub struct AdTemplatesExplainArgs { /// Model consent denying server-side auction. #[arg(long)] pub consent_denied: bool, - /// Model Fastly `edgezero_enabled=true`. - #[arg(long)] - pub edgezero_enabled: bool, } /// Run an ad-template CLI command. @@ -88,10 +99,22 @@ pub struct AdTemplatesExplainArgs { /// /// Returns a user-facing string when config loading, matching, or assertion /// checks fail. -pub fn run_ad_templates(args: &AdTemplatesCommand) -> Result<(), String> { +pub fn run_ad_templates(args: &AdTemplatesCommand) -> Result { let stdout = io::stdout(); let mut out = stdout.lock(); - run_ad_templates_with_writer(args, &mut out) + if let AdTemplatesCommand::Check(args) = args { + return match run_check_classified(args, &mut out) { + Ok(()) => Ok(RunOutcome::Success), + Err(CheckFailure::Tool(error)) => Err(error), + Err(CheckFailure::Assertion(message)) => { + let stderr = io::stderr(); + let mut err = stderr.lock(); + writeln!(err, "{message}").map_err(output_error)?; + Ok(RunOutcome::AssertionFailed) + } + }; + } + run_ad_templates_with_writer(args, &mut out).map(|()| RunOutcome::Success) } fn run_ad_templates_with_writer( @@ -171,12 +194,18 @@ fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), Strin .map_err(output_error)?; } - if !config.slot.is_empty() { - writeln!( - out, - "edgezero: configured slots currently require Fastly legacy fallback" - ) - .map_err(output_error)?; + for slot in &config.slot { + for pattern in &slot.page_patterns { + if let Err(error) = validate_page_pattern(pattern) { + writeln!( + out, + "invalid page pattern for slot `{}`: {}", + escape_terminal_text(&slot.id), + escape_terminal_text(&error), + ) + .map_err(output_error)?; + } + } } Ok(()) @@ -206,15 +235,17 @@ fn run_match(args: &AdTemplatesMatchArgs, out: &mut dyn Write) -> Result<(), Str } fn run_check(args: &AdTemplatesCheckArgs, out: &mut dyn Write) -> Result<(), String> { - if args.expect_no_slots && !args.expected_slots.is_empty() { - return Err("--expect-no-slots cannot be combined with --expected-slot".to_string()); - } - if !args.expect_no_slots && args.expected_slots.is_empty() { - return Err("provide --expected-slot at least once or pass --expect-no-slots".to_string()); - } + run_check_classified(args, out).map_err(|failure| match failure { + CheckFailure::Tool(error) | CheckFailure::Assertion(error) => error, + }) +} - let loaded = load_settings(&args.config)?; - let path = normalize_path_or_url(&args.path_or_url)?; +fn run_check_classified( + args: &AdTemplatesCheckArgs, + out: &mut dyn Write, +) -> Result<(), CheckFailure> { + let loaded = load_settings(&args.config).map_err(CheckFailure::Tool)?; + let path = normalize_path_or_url(&args.path_or_url).map_err(CheckFailure::Tool)?; let matched = loaded .settings .creative_opportunities @@ -225,13 +256,15 @@ fn run_check(args: &AdTemplatesCheckArgs, out: &mut dyn Write) -> Result<(), Str if args.expect_no_slots { if actual.is_empty() { - writeln!(out, "{path}: OK, no slots matched").map_err(output_error)?; + writeln!(out, "{path}: OK, no slots matched") + .map_err(output_error) + .map_err(CheckFailure::Tool)?; return Ok(()); } - return Err(format!( + return Err(CheckFailure::Assertion(format!( "{path}: expected no slots, matched {}", join_set(&actual) - )); + ))); } let expected: BTreeSet<&str> = args.expected_slots.iter().map(String::as_str).collect(); @@ -239,7 +272,9 @@ fn run_check(args: &AdTemplatesCheckArgs, out: &mut dyn Write) -> Result<(), Str let extra: BTreeSet<&str> = actual.difference(&expected).copied().collect(); if missing.is_empty() && (args.allow_extra_slots || extra.is_empty()) { - writeln!(out, "{path}: OK, matched {}", join_set(&actual)).map_err(output_error)?; + writeln!(out, "{path}: OK, matched {}", join_set(&actual)) + .map_err(output_error) + .map_err(CheckFailure::Tool)?; return Ok(()); } @@ -250,7 +285,10 @@ fn run_check(args: &AdTemplatesCheckArgs, out: &mut dyn Write) -> Result<(), Str if !args.allow_extra_slots && !extra.is_empty() { problems.push(format!("unexpected {}", join_set(&extra))); } - Err(format!("{path}: {}", problems.join("; "))) + Err(CheckFailure::Assertion(format!( + "{path}: {}", + problems.join("; ") + ))) } fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), String> { @@ -274,27 +312,13 @@ fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), true, )?; - let method_pass = args.method.eq_ignore_ascii_case("GET"); + let method_pass = args.method == Method::GET; let navigation_pass = !args.non_navigation; - let prefetch_pass = !args.prefetch; - let bot_pass = !args.bot; let consent_pass = !args.consent_denied; let auction_enabled = loaded.settings.auction.enabled; let providers_configured = !loaded.settings.auction.providers.is_empty(); let has_matches = !matched.is_empty(); - write_gate(out, "method GET", method_pass)?; - write_gate(out, "navigation", navigation_pass)?; - write_gate(out, "not prefetch", prefetch_pass)?; - write_gate(out, "not bot", bot_pass)?; - write_gate(out, "consent allows auction", consent_pass)?; - write_gate(out, "auction.enabled", auction_enabled)?; - write_gate(out, "auction providers configured", providers_configured)?; - write_gate(out, "matched slots", has_matches)?; - - // Share the runtime ad-stack decision with `publisher.rs` so explain cannot - // drift from the live gate. The "auction providers configured" gate is an - // explain-only supplementary check the runtime helper intentionally omits. let gate = evaluate_ad_stack_gate(AdStackGateInput { method_get: method_pass, navigation: navigation_pass, @@ -304,22 +328,55 @@ fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), consent_allows_auction: Some(consent_pass), auction_enabled, }); - let runs_ad_stack = gate.expected == RuntimeAdStackExpected::Yes && providers_configured; + let blocked: Vec = gate.blocking_gates().collect(); + write_gate( + out, + "method GET", + !blocked.contains(&AdStackGateName::MethodGet), + )?; + write_gate( + out, + "navigation", + !blocked.contains(&AdStackGateName::Navigation), + )?; + write_gate( + out, + "not prefetch", + !blocked.contains(&AdStackGateName::NotPrefetch), + )?; + write_gate(out, "not bot", !blocked.contains(&AdStackGateName::NotBot))?; + write_gate( + out, + "consent allows auction", + !blocked.contains(&AdStackGateName::ConsentAllowsAuction), + )?; + write_gate( + out, + "auction.enabled", + !blocked.contains(&AdStackGateName::AuctionEnabled), + )?; + write_gate( + out, + "matched slots", + !blocked.contains(&AdStackGateName::MatchedSlots), + )?; + writeln!( + out, + "advisory auction providers configured: {}", + if providers_configured { "yes" } else { "no" } + ) + .map_err(output_error)?; writeln!( out, "server-side ad stack: {}", - if runs_ad_stack { "yes" } else { "no" } + match gate.expected { + RuntimeAdStackExpected::Yes => "yes", + RuntimeAdStackExpected::No => "no", + RuntimeAdStackExpected::Unknown => "unknown", + } ) .map_err(output_error)?; - if args.edgezero_enabled && !config.slot.is_empty() { - writeln!( - out, - "edgezero: configured slots require Fastly legacy fallback until buffered EdgeZero ad-template injection is wired" - ) - .map_err(output_error)?; - } - Ok(()) } @@ -562,10 +619,9 @@ mod tests { } #[test] - fn explain_reports_runtime_gates_and_edgezero_fallback() { - let config_text = config_with_slots() - .replace("[auction]\nenabled = false", "[auction]\nenabled = true") - .replace("providers = []", "providers = [\"prebid\"]"); + fn explain_keeps_provider_state_separate_from_runtime_verdict() { + let config_text = + config_with_slots().replace("[auction]\nenabled = false", "[auction]\nenabled = true"); let (_temp, config) = project_with_config(&config_text); let mut out = Vec::new(); @@ -573,12 +629,11 @@ mod tests { &AdTemplatesCommand::Explain(AdTemplatesExplainArgs { config, path_or_url: "/news/story".to_string(), - method: "GET".to_string(), + method: Method::GET, non_navigation: false, prefetch: false, bot: false, consent_denied: false, - edgezero_enabled: true, }), &mut out, ) @@ -587,11 +642,11 @@ mod tests { let output = String::from_utf8(out).expect("should be utf8"); assert!( output.contains("server-side ad stack: yes"), - "should report ad stack enabled" + "runtime verdict should not include provider configuration" ); assert!( - output.contains("configured slots require Fastly legacy fallback"), - "should report EdgeZero fallback" + output.contains("advisory auction providers configured: no"), + "provider state should be a separate advisory" ); } @@ -615,9 +670,45 @@ mod tests { output.contains("auction.enabled:"), "should report the auction kill-switch state" ); + assert!(!output.contains("legacy fallback")); + } + + #[test] + fn lint_reports_page_patterns_the_runtime_drops() { + let config_text = config_with_slots().replace( + "page_patterns = [\"/news/*\", \"/\"]", + "page_patterns = [\"/news/*\", \"[\"]", + ); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { config }), + &mut out, + ) + .expect("should lint mixed valid and invalid patterns"); + let output = String::from_utf8(out).expect("should be utf8"); + assert!( - output.contains("edgezero: configured slots currently require Fastly legacy fallback"), - "should report the EdgeZero legacy-fallback note" + output.contains("invalid page pattern for slot `atf`") + && output.contains("page pattern '[' is not a valid glob"), + "lint should surface the runtime-dropped pattern: {output}" ); } + + #[test] + fn public_check_reports_drift_as_assertion_outcome() { + let (_temp, config) = project_with_config(&config_with_slots()); + + let outcome = run_ad_templates(&AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/sports/game".to_string(), + expected_slots: vec!["atf".to_string()], + expect_no_slots: false, + allow_extra_slots: false, + })) + .expect("assertion drift should not be a tool error"); + + assert_eq!(outcome, RunOutcome::AssertionFailed); + } } diff --git a/crates/trusted-server-cli/src/lib.rs b/crates/trusted-server-cli/src/lib.rs index bc3a970a9..471813683 100644 --- a/crates/trusted-server-cli/src/lib.rs +++ b/crates/trusted-server-cli/src/lib.rs @@ -22,7 +22,7 @@ mod prebid_bundle; mod run; #[cfg(not(target_arch = "wasm32"))] -pub use run::run_from_env; +pub use run::{RunOutcome, run_from_env}; // Every `ts` subcommand's implementation lives under `commands/`. The // `ts dev` group is available on every host target; its only subcommand, diff --git a/crates/trusted-server-cli/src/main.rs b/crates/trusted-server-cli/src/main.rs index 7cee5b1ca..9cf72215a 100644 --- a/crates/trusted-server-cli/src/main.rs +++ b/crates/trusted-server-cli/src/main.rs @@ -3,9 +3,13 @@ fn main() { use std::process; edgezero_cli::init_cli_logger(); - if let Err(err) = trusted_server_cli::run_from_env() { - log::error!("[ts] {err}"); - process::exit(2); + match trusted_server_cli::run_from_env() { + Ok(outcome) if outcome.exit_code() != 0 => process::exit(outcome.exit_code()), + Ok(_) => {} + Err(err) => { + log::error!("[ts] {err}"); + process::exit(2); + } } } diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 9aade197e..d39ce2180 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -70,37 +70,62 @@ enum PrebidCommand { Bundle(PrebidBundleArgs), } +/// Process-level outcome for commands that distinguish drift from tool errors. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RunOutcome { + /// Command completed without drift. + Success, + /// Command ran successfully and found assertion drift. + AssertionFailed, +} + +impl RunOutcome { + /// Stable process exit code for this outcome. + #[must_use] + pub const fn exit_code(self) -> i32 { + match self { + Self::Success => 0, + Self::AssertionFailed => 1, + } + } +} + /// Run the CLI using process arguments. /// /// # Errors /// /// Returns an error when command parsing, config validation, `EdgeZero` /// delegation, audit collection, config initialization, or Prebid bundle generation fails. -pub fn run_from_env() -> Result<(), String> { +pub fn run_from_env() -> Result { dispatch(Args::parse()) } -fn dispatch(args: Args) -> Result<(), String> { +fn dispatch(args: Args) -> Result { match args.command { - Command::Auth(args) => edgezero_cli::run_auth(&args), + Command::Auth(args) => edgezero_cli::run_auth(&args).map(|()| RunOutcome::Success), Command::Audit(args) => run_audit(&args), - Command::Build(args) => edgezero_cli::run_build(&args), + Command::Build(args) => edgezero_cli::run_build(&args).map(|()| RunOutcome::Success), Command::Config(ConfigCommand::AdTemplates(args)) => run_ad_templates(&args), - Command::Config(ConfigCommand::Init(args)) => run_config_init(&args), + Command::Config(ConfigCommand::Init(args)) => { + run_config_init(&args).map(|()| RunOutcome::Success) + } Command::Config(ConfigCommand::Diff(args)) => { match edgezero_cli::run_config_diff_typed::(&args) { - Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(()), + Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(RunOutcome::Success), + Ok(edgezero_cli::DiffExit { code: 1 }) => Ok(RunOutcome::AssertionFailed), Ok(edgezero_cli::DiffExit { code }) => process::exit(code), Err(err) => Err(err), } } Command::Config(ConfigCommand::Push(args)) => { edgezero_cli::run_config_push_typed::(&args) + .map(|()| RunOutcome::Success) } Command::Config(ConfigCommand::Validate(args)) => { edgezero_cli::run_config_validate_typed::(&args) + .map(|()| RunOutcome::Success) } - Command::Deploy(args) => edgezero_cli::run_deploy(&args), + Command::Deploy(args) => edgezero_cli::run_deploy(&args).map(|()| RunOutcome::Success), Command::Prebid(prebid) => { let mut generator = NpmPrebidBundleGenerator; let mut stdout = std::io::stdout(); @@ -108,12 +133,15 @@ fn dispatch(args: Args) -> Result<(), String> { match prebid.command { PrebidCommand::Bundle(args) => { run_bundle(&args, &mut generator, &mut stdout, &mut stderr) + .map(|()| RunOutcome::Success) } } } - Command::Provision(args) => edgezero_cli::run_provision(&args), - Command::Serve(args) => edgezero_cli::run_serve(&args), - Command::Dev(command) => crate::commands::dev::run(command), + Command::Provision(args) => { + edgezero_cli::run_provision(&args).map(|()| RunOutcome::Success) + } + Command::Serve(args) => edgezero_cli::run_serve(&args).map(|()| RunOutcome::Success), + Command::Dev(command) => crate::commands::dev::run(command).map(|()| RunOutcome::Success), } } @@ -130,6 +158,12 @@ mod tests { Args::try_parse_from(args).expect("should parse args") } + #[test] + fn run_outcomes_use_documented_exit_codes() { + assert_eq!(RunOutcome::Success.exit_code(), 0); + assert_eq!(RunOutcome::AssertionFailed.exit_code(), 1); + } + #[test] fn parses_build_with_adapter_args() { let args = parse(&[ @@ -285,6 +319,52 @@ mod tests { assert!(!check_args.expect_no_slots); } + #[test] + fn config_ad_templates_check_requires_an_expectation_mode() { + assert!(Args::try_parse_from(["ts", "config", "ad-templates", "check", "/news"]).is_err()); + } + + #[test] + fn config_ad_templates_check_rejects_extra_slots_with_no_slots_mode() { + assert!( + Args::try_parse_from([ + "ts", + "config", + "ad-templates", + "check", + "/news", + "--expect-no-slots", + "--allow-extra-slots", + ]) + .is_err() + ); + } + + #[test] + fn config_ad_templates_explain_rejects_removed_edgezero_model() { + assert!( + Args::try_parse_from([ + "ts", + "config", + "ad-templates", + "explain", + "/news", + "--edgezero-enabled", + ]) + .is_err() + ); + } + + #[test] + fn bare_audit_namespace_displays_help_as_an_error() { + let error = Args::try_parse_from(["ts", "audit"]).expect_err("should require audit mode"); + + assert_eq!( + error.kind(), + clap::error::ErrorKind::DisplayHelpOnMissingArgumentOrSubcommand + ); + } + #[test] fn audit_legacy_url_parses_with_artifact_generation_flags() { let args = parse(&[ From e8fb2eef97d724ffc7eafb2cc4a46c33f75d7a83 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 18 Aug 2026 14:28:23 +0530 Subject: [PATCH 317/395] Bound browser ad template evidence collection --- .../commands/audit/ad_template_collector.js | 123 +++++++++++------ .../src/commands/audit/browser.rs | 129 ++++++++++++++---- .../src/commands/audit/collector.rs | 24 +++- 3 files changed, 204 insertions(+), 72 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js index 1133d46b6..ee610595d 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -1,4 +1,4 @@ -// Read-only ad-template evidence collector, injected before publisher scripts run. +// Bounded ad-template evidence collector, injected before publisher scripts run. // // This body runs inside an IIFE that defines `__TS_CONFIG` (configured div // prefixes + APS slot IDs). It records evidence into `window.__tsAdTemplateEvidence` @@ -21,17 +21,29 @@ const __ts_phase = () => (window.__tsScrollPhase ? "scroll" : "initial_load") // Hard cap per evidence list so a hostile page cannot grow the store without // bound; the page controls how many slots/elements/warnings it produces. const __ts_max_entries = 1024 +const __ts_max_string_length = 512 +const __ts_wrapped_googletags = new WeakSet() +const __ts_wrapped_apstags = new WeakSet() + +function __ts_text(value) { + return String(value).slice(0, __ts_max_string_length) +} + function __ts_push(list, entry) { if (list.length < __ts_max_entries) list.push(entry) } +function __ts_warn(code, error) { + __ts_push(__ts_ev.warnings, { code, message: __ts_text(error) }) +} + // GPT sizes reach Rust as u32 pairs, so anything non-integral (fluid slots, // NaN, negative or fractional dimensions) must be dropped here — a single bad // pair would fail deserialization of the whole evidence payload and discard // every other slot's otherwise valid evidence. function __ts_size_pair(width, height) { if (!Number.isInteger(width) || !Number.isInteger(height)) return null - if (width < 0 || height < 0) return null + if (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) return null return [width, height] } @@ -57,58 +69,74 @@ function __ts_normalize_sizes(sizes) { function __ts_record_define_slot(adUnitPath, sizes, divId) { __ts_push(__ts_ev.gpt_slots, { - gam_unit_path: String(adUnitPath), - div_id: String(divId), + gam_unit_path: __ts_text(adUnitPath), + div_id: __ts_text(divId), sizes: __ts_normalize_sizes(sizes), phase: __ts_phase(), }) } function __ts_wrap_googletag(googletag) { - if (!googletag || googletag.__tsWrapped) return googletag - googletag.__tsWrapped = true - googletag.cmd = googletag.cmd || [] - // Wrap cmd.push without changing callback order (pass-through to the original). - const originalPush = googletag.cmd.push.bind(googletag.cmd) - googletag.cmd.push = function (callback) { - return originalPush(callback) + if (!googletag || (typeof googletag !== "object" && typeof googletag !== "function")) { + return googletag } + if (__ts_wrapped_googletags.has(googletag)) return googletag + __ts_wrapped_googletags.add(googletag) // Wrap defineSlot so both direct calls and calls dispatched from the cmd queue // are recorded (queued callbacks call this same wrapped function). const originalDefineSlot = googletag.defineSlot if (typeof originalDefineSlot === "function") { - googletag.defineSlot = function (adUnitPath, sizes, divId) { - const slot = originalDefineSlot.apply(this, arguments) - try { - __ts_record_define_slot(adUnitPath, sizes, divId) - } catch (error) { - __ts_push(__ts_ev.warnings, { code: "define_slot_capture_failed", message: String(error) }) - } - return slot + try { + Object.defineProperty(googletag, "defineSlot", { + configurable: true, + enumerable: false, + writable: true, + value: function (adUnitPath, sizes, divId) { + const slot = originalDefineSlot.apply(this, arguments) + try { + __ts_record_define_slot(adUnitPath, sizes, divId) + } catch (error) { + __ts_warn("define_slot_capture_failed", error) + } + return slot + }, + }) + } catch (error) { + __ts_warn("define_slot_wrap_failed", error) } } return googletag } function __ts_wrap_apstag(apstag) { - if (!apstag || apstag.__tsWrapped) return apstag - apstag.__tsWrapped = true + if (!apstag || (typeof apstag !== "object" && typeof apstag !== "function")) return apstag + if (__ts_wrapped_apstags.has(apstag)) return apstag + __ts_wrapped_apstags.add(apstag) const originalFetchBids = apstag.fetchBids if (typeof originalFetchBids === "function") { - apstag.fetchBids = function (config, callback) { - try { - const slots = (config && config.slots) || [] - for (const slot of slots) { - __ts_push(__ts_ev.aps_calls, { - slot_id: String(slot.slotID || slot.slotName || ""), - sizes: __ts_normalize_sizes(slot.sizes), - phase: __ts_phase(), - }) - } - } catch (error) { - __ts_push(__ts_ev.warnings, { code: "aps_capture_failed", message: String(error) }) - } - return originalFetchBids.apply(this, arguments) + try { + Object.defineProperty(apstag, "fetchBids", { + configurable: true, + enumerable: false, + writable: true, + value: function (config, callback) { + try { + const slots = (config && config.slots) || [] + for (const slot of slots) { + __ts_push(__ts_ev.aps_calls, { + slot_id: __ts_text(slot.slotID || slot.slotName || ""), + sizes: __ts_normalize_sizes(slot.sizes), + phase: __ts_phase(), + }) + } + } catch (error) { + __ts_warn("aps_capture_failed", error) + } + return originalFetchBids.apply(this, arguments) + }, + }) + } catch (error) { + __ts_warn("aps_wrap_failed", error) } } return apstag @@ -117,7 +145,11 @@ function __ts_wrap_apstag(apstag) { // Wrap an existing global or intercept a later assignment of it. function __ts_install(name, wrap) { if (window[name]) { - wrap(window[name]) + try { + wrap(window[name]) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } return } let internal @@ -127,7 +159,12 @@ function __ts_install(name, wrap) { return internal }, set(value) { - internal = wrap(value) + internal = value + try { + internal = wrap(value) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } }, }) } @@ -140,7 +177,7 @@ window.__tsCollectAdTemplateEvidence = function () { try { const seen = new Set(__ts_ev.dom_ids.map((entry) => entry.dom_id)) for (const element of document.querySelectorAll("[id]")) { - const id = element.id + const id = __ts_text(element.id) if (id.endsWith("-container")) continue if (__ts_prefixes.some((prefix) => id.startsWith(prefix)) && !seen.has(id)) { __ts_push(__ts_ev.dom_ids, { dom_id: id, phase: __ts_phase() }) @@ -177,23 +214,23 @@ window.__tsCollectAdTemplateEvidence = function () { } } const exists = __ts_ev.gpt_slots.some( - (entry) => entry.gam_unit_path === String(path) && entry.div_id === String(divId) + (entry) => entry.gam_unit_path === __ts_text(path) && entry.div_id === __ts_text(divId) ) if (!exists) { __ts_push(__ts_ev.gpt_slots, { - gam_unit_path: String(path), - div_id: String(divId), + gam_unit_path: __ts_text(path), + div_id: __ts_text(divId), sizes, phase: __ts_phase(), }) } } catch (error) { - __ts_push(__ts_ev.warnings, { code: "gpt_scrape_failed", message: String(error) }) + __ts_warn("gpt_scrape_failed", error) } } } } catch (error) { - __ts_push(__ts_ev.warnings, { code: "collect_failed", message: String(error) }) + __ts_warn("collect_failed", error) } return __ts_ev } diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs index ab998adec..d0b656770 100644 --- a/crates/trusted-server-cli/src/commands/audit/browser.rs +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -36,6 +36,8 @@ const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); /// Hard cap per decoded evidence list, mirroring the collector script's /// `__ts_max_entries`, so a hostile page cannot inflate CLI memory. const MAX_EVIDENCE_ENTRIES: usize = 1024; +/// Hard cap on the UTF-8 JSON payload before CDP transfers it back to Rust. +const MAX_EVIDENCE_PAYLOAD_BYTES: usize = 1024 * 1024; /// Hard cap on browser teardown so a wedged Chrome cannot hang the audit. const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); /// Default quiet window (no new resources) marking the page settled. @@ -410,43 +412,101 @@ async fn extract_ad_evidence( page: &Page, warnings: &mut Vec, ) -> Option { - // Trigger the on-demand DOM + getSlots scrape, then read the evidence object. - let value = page - .evaluate( - "(typeof window.__tsCollectAdTemplateEvidence === 'function' \ - ? window.__tsCollectAdTemplateEvidence() \ - : (window.__tsAdTemplateEvidence || null))", - ) + // Serialize and size-check in the page so a hostile publisher-controlled + // evidence object cannot force an unbounded CDP response and Rust decode. + let envelope = page + .evaluate(format!( + r#"(() => {{ + const evidence = typeof window.__tsCollectAdTemplateEvidence === 'function' + ? window.__tsCollectAdTemplateEvidence() + : (window.__tsAdTemplateEvidence || null) + if (evidence === null) return {{ kind: 'absent' }} + try {{ + const json = JSON.stringify(evidence) + const bytes = new TextEncoder().encode(json).byteLength + if (bytes > {MAX_EVIDENCE_PAYLOAD_BYTES}) return {{ kind: 'too_large' }} + return {{ kind: 'evidence', json }} + }} catch (error) {{ + return {{ + kind: 'serialization_failed', + message: String(error).slice(0, 512), + }} + }} + }})()"# + )) .await .ok() - .and_then(|result| result.into_value::().ok()); + .and_then(|result| result.into_value::().ok()); - match value { - Some(serde_json::Value::Null) | None => { + match envelope { + Some(envelope) => decode_ad_evidence_envelope(envelope, warnings), + None => { warnings.push(Warning { code: "ad_evidence_absent".to_string(), message: "no ad-template evidence was collected from the page".to_string(), }); None } - Some(value) => match serde_json::from_value::(value) { - Ok(mut evidence) => { - // Defense in depth: the injected script caps these lists, but the - // page owns that store, so re-cap after decode. - evidence.dom_ids.truncate(MAX_EVIDENCE_ENTRIES); - evidence.gpt_slots.truncate(MAX_EVIDENCE_ENTRIES); - evidence.aps_calls.truncate(MAX_EVIDENCE_ENTRIES); - evidence.warnings.truncate(MAX_EVIDENCE_ENTRIES); - Some(evidence) - } - Err(error) => { - warnings.push(Warning { - code: "ad_evidence_decode_failed".to_string(), - message: format!("failed to decode ad-template evidence: {error}"), - }); - None + } +} + +#[derive(Debug, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum EvidenceEnvelope { + Absent, + TooLarge, + Evidence { json: String }, + SerializationFailed { message: String }, +} + +fn decode_ad_evidence_envelope( + envelope: EvidenceEnvelope, + warnings: &mut Vec, +) -> Option { + match envelope { + EvidenceEnvelope::Absent => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + EvidenceEnvelope::TooLarge => { + warnings.push(Warning { + code: "ad_evidence_too_large".to_string(), + message: format!( + "ad-template evidence exceeded the {MAX_EVIDENCE_PAYLOAD_BYTES}-byte limit" + ), + }); + None + } + EvidenceEnvelope::SerializationFailed { message } => { + warnings.push(Warning { + code: "ad_evidence_encode_failed".to_string(), + message: format!("failed to serialize ad-template evidence in the page: {message}"), + }); + None + } + EvidenceEnvelope::Evidence { json } => { + match serde_json::from_str::(&json) { + Ok(mut evidence) => { + // Defense in depth: the injected script caps these lists, but the + // page owns that store, so re-cap after decode. + evidence.dom_ids.truncate(MAX_EVIDENCE_ENTRIES); + evidence.gpt_slots.truncate(MAX_EVIDENCE_ENTRIES); + evidence.aps_calls.truncate(MAX_EVIDENCE_ENTRIES); + evidence.warnings.truncate(MAX_EVIDENCE_ENTRIES); + Some(evidence) + } + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence: {error}"), + }); + None + } } - }, + } } } @@ -473,6 +533,16 @@ mod tests { ); } + #[test] + fn oversized_ad_evidence_is_an_explicit_warning() { + let mut warnings = Vec::new(); + let evidence = decode_ad_evidence_envelope(EvidenceEnvelope::TooLarge, &mut warnings); + + assert!(evidence.is_none()); + assert_eq!(warnings.len(), 1); + assert_eq!(warnings[0].code, "ad_evidence_too_large"); + } + /// A self-contained page that stubs just enough of GPT (no network) for the /// collector to observe a defined slot via the wrapped `defineSlot` and the /// `getSlots()` scrape. @@ -483,6 +553,9 @@ mod tests {
+"#; /// Configuration for the APS `OpenRTB` integration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] @@ -1152,28 +1211,13 @@ impl IntegrationProxy for ApsRendererIntegration { message: "Failed to build APS not-found response".to_string(), }); } - let (renderer_document, renderer_csp) = match request.uri().query() { - None => (APS_RENDERER_DOCUMENT, APS_RENDERER_CSP), - Some(APS_RENDERER_BOOTSTRAP_QUERY) => { - (APS_RENDERER_BOOTSTRAP_DOCUMENT, APS_RENDERER_BOOTSTRAP_CSP) - } - Some(_) => { - return http::Response::builder() - .status(StatusCode::NOT_FOUND) - .body(EdgeBody::from("Not Found")) - .change_context(TrustedServerError::Integration { - integration: APS_INTEGRATION_ID.to_string(), - message: "Failed to build APS not-found response".to_string(), - }); - } - }; http::Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "text/html; charset=utf-8") .header("x-content-type-options", "nosniff") .header("referrer-policy", "no-referrer") - .header(header::CONTENT_SECURITY_POLICY, renderer_csp) - .body(EdgeBody::from(renderer_document)) + .header(header::CONTENT_SECURITY_POLICY, APS_RENDERER_CSP) + .body(EdgeBody::from(APS_RENDERER_DOCUMENT)) .change_context(TrustedServerError::Integration { integration: APS_INTEGRATION_ID.to_string(), message: "Failed to build APS renderer response".to_string(), @@ -2274,7 +2318,7 @@ mod tests { } #[test] - fn registers_and_serves_static_renderer_and_data_bootstrap_modes() { + fn registers_and_serves_only_static_renderer_route() { let integration = ApsRendererIntegration; let routes = integration.routes(); assert_eq!(routes.len(), 1, "should register one route"); @@ -2303,25 +2347,6 @@ mod tests { APS_RENDERER_CSP ); - let bootstrap = http::Request::builder() - .method(Method::GET) - .uri(format!( - "{APS_RENDERER_ROUTE}?{APS_RENDERER_BOOTSTRAP_QUERY}" - )) - .body(EdgeBody::empty()) - .expect("should build renderer bootstrap request"); - let response = - futures::executor::block_on(integration.handle(&settings, &services, bootstrap)) - .expect("should serve renderer bootstrap"); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - response.headers()[header::CONTENT_SECURITY_POLICY], - APS_RENDERER_BOOTSTRAP_CSP - ); - assert!(APS_RENDERER_BOOTSTRAP_CSP.contains("allow-same-origin")); - assert!(APS_RENDERER_BOOTSTRAP_CSP.contains("frame-src https: data:")); - assert!(APS_RENDERER_BOOTSTRAP_DOCUMENT.contains("trusted-server/aps/bootstrap-navigate")); - let post = http::Request::builder() .method(Method::POST) .uri(APS_RENDERER_ROUTE) @@ -2349,7 +2374,6 @@ mod tests { assert_eq!(registration.integration_id, APS_INTEGRATION_ID); assert_eq!(registration.proxies.len(), 1); - assert!(registration.request_filters.is_empty()); assert!(registration.js_disabled); } @@ -2401,15 +2425,12 @@ mod tests { #[test] fn renderer_document_is_static_and_nonce_bound() { assert!(APS_RENDERER_DOCUMENT.contains("^#tsaps=")); - assert!(APS_RENDERER_DOCUMENT.contains("event.source !== parent")); - assert!(APS_RENDERER_DOCUMENT.contains("message.nonce !== expected")); - assert!(APS_RENDERER_DOCUMENT.contains("['nonce', 'publisherOrigin', 'renderer']")); - assert!(APS_RENDERER_DOCUMENT.contains("['nonce', 'renderer']")); - assert!(APS_RENDERER_DOCUMENT.contains("url.origin === publisherOrigin")); + assert!(APS_RENDERER_DOCUMENT.contains("event.source!==parent")); + assert!(APS_RENDERER_DOCUMENT.contains("message.nonce!==expected")); assert!(APS_RENDERER_DOCUMENT.contains("prebid/creative/render")); assert!(APS_RENDERER_DOCUMENT.contains("window._aps instanceof Map")); - assert!(APS_RENDERER_DOCUMENT.contains("store: new Map([['listeners', new Map()]])")); - assert!(APS_RENDERER_DOCUMENT.contains("account.queue.push(")); + assert!(APS_RENDERER_DOCUMENT.contains("store:new Map([['listeners',new Map()]])")); + assert!(APS_RENDERER_DOCUMENT.contains("account.queue.push(new CustomEvent")); assert!( APS_RENDERER_DOCUMENT.contains("trusted-server/aps/renderer-ready") && APS_RENDERER_DOCUMENT.contains("trusted-server/aps/renderer-failed") @@ -2421,7 +2442,7 @@ mod tests { ); assert!(!APS_RENDERER_DOCUMENT.contains(" diff --git a/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html b/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html deleted file mode 100644 index 6ef871c9b..000000000 --- a/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - - - diff --git a/crates/trusted-server-js/lib/src/integrations/aps/renderer.html b/crates/trusted-server-js/lib/src/integrations/aps/renderer.html deleted file mode 100644 index b7ddac1c0..000000000 --- a/crates/trusted-server-js/lib/src/integrations/aps/renderer.html +++ /dev/null @@ -1,228 +0,0 @@ - - - - - diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index ef6a3204c..52a6ecb70 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -10,9 +10,9 @@ import type { import { APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, + apsRendererUrl, consumeApsPrebidRenderer, getApsPrebidRenderer, - registerApsUniversalCreativeMount, validateApsRenderer, } from '../aps/render'; @@ -225,14 +225,13 @@ function slotIdForMessageSource(source: MessageEventSource | null): string | und ?.id; } -function slotRootForMessageSource( +function messageSourceBelongsToAdUnit( source: MessageEventSource | null, - divId: string -): HTMLElement | undefined { - if (!source) return undefined; - return candidateSlotRootsForConfiguredDivId(divId).find((root) => - sourceIsInSlotRoots(source, [root]) - ); + adUnitCode: string +): boolean { + return source + ? sourceIsInSlotRoots(source, candidateSlotRootsForConfiguredDivId(adUnitCode)) + : false; } function clearTargetingKeys(slot: GoogleTagSlot, keys: Iterable): void { @@ -1701,15 +1700,13 @@ export function installTsRenderBridge(): void { // Prebid handles ad IDs globally and would otherwise answer a request from // an unrelated iframe when this slot-bound capability rejects it. e.stopImmediatePropagation(); - const mountContainer = slotRootForMessageSource(e.source, prebidRendererEntry.adUnitCode); - if (!mountContainer) return; + if (!messageSourceBelongsToAdUnit(e.source, prebidRendererEntry.adUnitCode)) return; const renderer = validateApsRenderer(prebidRendererEntry.renderer); - if (!renderer) return; + const rendererUrl = apsRendererUrl(); + if (!renderer || !rendererUrl) return; if (!hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; if (!consumeApsPrebidRenderer(adId, prebidRendererEntry)) return; recordConsumedPrebidApsId(consumedPrebidApsIds, adId, prebidRendererEntry.expiresAt); - const apsMountId = registerApsUniversalCreativeMount(mountContainer, renderer); - if (!apsMountId) return; port.postMessage( JSON.stringify({ @@ -1717,8 +1714,7 @@ export function installTsRenderBridge(): void { adId, renderer: APS_UNIVERSAL_CREATIVE_RENDERER, rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - apsMountId, - publisherOrigin: window.location.origin, + rendererUrl, apsRenderer: renderer, width: renderer.width, height: renderer.height, @@ -1757,11 +1753,8 @@ export function installTsRenderBridge(): void { e.stopImmediatePropagation(); if (consumedServerApsBySlot.get(slotId) === adId) return; const renderer = validateApsRenderer(matchedBid.renderer); - const configuredSlot = window.tsjs?.adSlots?.find((slot) => slot.id === slotId); - const mountContainer = slotRootForMessageSource(e.source, configuredSlot?.div_id ?? slotId); - if (!renderer || !mountContainer) return; - const apsMountId = registerApsUniversalCreativeMount(mountContainer, renderer); - if (!apsMountId) return; + const rendererUrl = apsRendererUrl(); + if (!renderer || !rendererUrl) return; consumedServerApsBySlot.set(slotId, adId); port.postMessage( JSON.stringify({ @@ -1769,8 +1762,7 @@ export function installTsRenderBridge(): void { adId, renderer: APS_UNIVERSAL_CREATIVE_RENDERER, rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - apsMountId, - publisherOrigin: window.location.origin, + rendererUrl, apsRenderer: renderer, width: renderer.width, height: renderer.height, diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index ba2dfb274..dc17c9e87 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -69,7 +69,7 @@ describe('request.requestAds', () => { ); }); - it('dispatches a valid APS descriptor through the opaque data renderer bootstrap', async () => { + it('dispatches a valid APS descriptor to the opaque static renderer route', async () => { const apsBid = envelope.seatbid[0].bid[0]; const renderer = { type: 'aps', @@ -117,55 +117,21 @@ describe('request.requestAds', () => { const iframe = document.querySelector('#slot1 iframe') as HTMLIFrameElement | null; expect(iframe).not.toBeNull(); - expect(iframe!.src).toContain('/integrations/aps/renderer?mode=data-bootstrap#tsaps='); + expect(iframe!.src).toContain('/integrations/aps/renderer#tsaps='); expect(iframe!.srcdoc).toBe(''); expect(iframe!.getAttribute('sandbox')).not.toContain('allow-same-origin'); expect(document.querySelector('#slot1 span')).not.toBeNull(); const postMessage = vi.spyOn(iframe!.contentWindow!, 'postMessage'); - const nonce = new URL(iframe!.src).hash.replace('#tsaps=', ''); - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/bootstrap-ready', nonce }, - source: iframe!.contentWindow, - }) - ); - const navigate = postMessage.mock.calls[0][0] as { rendererUrl: string }; - const containerDocument = decodeURIComponent( - navigate.rendererUrl - .slice('data:text/html;charset=utf-8,'.length) - .replace(/#tsaps=[A-Za-z0-9_-]{22}$/, '') - ); - const innerNonce = containerDocument.match( - /data:text\/html;charset=utf-8,.+#tsaps=([A-Za-z0-9_-]{22})/ - )?.[1]; - expect(innerNonce).toMatch(/^[A-Za-z0-9_-]{22}$/); - - const channel = { - close: vi.fn(), - onmessage: null, - postMessage: vi.fn(), - start: vi.fn(), - } as unknown as MessagePort; - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/container-ready', nonce }, - source: iframe!.contentWindow, - ports: [channel], - }) - ); - channel.onmessage?.( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/channel-ready', nonce: innerNonce }, - }) - ); + iframe!.dispatchEvent(new Event('load')); expect(document.querySelector('#slot1 span')).not.toBeNull(); - expect(channel.postMessage).toHaveBeenCalledWith(expect.objectContaining({ renderer })); + expect(postMessage).toHaveBeenCalledWith(expect.objectContaining({ renderer }), '*'); - const message = vi.mocked(channel.postMessage).mock.calls[0][0] as { nonce: string }; - channel.onmessage?.( + const message = postMessage.mock.calls[0][0] as { nonce: string }; + window.dispatchEvent( new MessageEvent('message', { data: { message: 'trusted-server/aps/renderer-ready', nonce: message.nonce }, + source: iframe!.contentWindow, }) ); expect(document.querySelector('#slot1 span')).toBeNull(); diff --git a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts index c9808b8ac..eae60c90e 100644 --- a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts @@ -4,18 +4,14 @@ import envelope from '../../fixtures/aps-renderer-v1.json'; import type { ApsRendererV1 } from '../../../src/core/types'; import { log } from '../../../src/core/log'; import { - APS_RENDERER_DATA_URL, APS_RENDERER_PATH, APS_RENDERER_SANDBOX, APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - apsRendererBootstrapUrl, apsRendererUrl, - cancelPendingApsRender, getApsPrebidRenderer, parseApsRendererDescriptor, registerApsPrebidRenderer, - registerApsUniversalCreativeMount, renderApsCreative, validateApsRenderer, } from '../../../src/integrations/aps/render'; @@ -54,76 +50,6 @@ function descriptor(overrides: Partial = {}): ApsRendererV1 { }; } -type FakeRendererChannel = MessagePort & { - close: ReturnType; - postMessage: ReturnType; - start: ReturnType; -}; - -function sendRendererMessage(channel: FakeRendererChannel, data: Record): void { - channel.onmessage?.(new MessageEvent('message', { data })); -} - -function advanceRendererToData(iframe: HTMLIFrameElement) { - const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); - const nonce = new URL(iframe.src).hash.replace('#tsaps=', ''); - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/bootstrap-ready', nonce }, - source: iframe.contentWindow, - }) - ); - expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); - const navigate = postMessage.mock.calls[0][0] as { - message: string; - nonce: string; - rendererUrl: string; - }; - expect(navigate.message).toBe('trusted-server/aps/bootstrap-navigate'); - expect(navigate.nonce).toBe(nonce); - expect(navigate.rendererUrl).toMatch( - /^data:text\/html;charset=utf-8,.+#tsaps=[A-Za-z0-9_-]{22}$/ - ); - - const encodedDocument = navigate.rendererUrl - .slice('data:text/html;charset=utf-8,'.length) - .replace(/#tsaps=[A-Za-z0-9_-]{22}$/, ''); - const containerDocument = decodeURIComponent(encodedDocument); - const innerNonce = containerDocument.match( - /data:text\/html;charset=utf-8,.+#tsaps=([A-Za-z0-9_-]{22})/ - )?.[1]; - expect(innerNonce).toMatch(/^[A-Za-z0-9_-]{22}$/); - expect(innerNonce).not.toBe(nonce); - expect(containerDocument).toContain('frame-src data: https://creative.example'); - expect(containerDocument).not.toContain(descriptor().bidId); - expect(containerDocument).not.toContain(descriptor().aaxResponse); - - const channel = { - close: vi.fn(), - onmessage: null, - postMessage: vi.fn(), - start: vi.fn(), - } as unknown as FakeRendererChannel; - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/container-ready', nonce }, - source: iframe.contentWindow, - ports: [channel], - }) - ); - expect(channel.start).toHaveBeenCalledOnce(); - sendRendererMessage(channel, { - message: 'trusted-server/aps/channel-ready', - nonce: innerNonce, - }); - const sent = channel.postMessage.mock.calls[0][0] as { - nonce: string; - publisherOrigin: string; - renderer: ApsRendererV1; - }; - return { channel, innerNonce: innerNonce!, postMessage, sent }; -} - describe('APS renderer validation', () => { it('consumes the shared fictional golden envelope and supports an omitted creative ID', () => { const withCreativeId = descriptor(); @@ -345,58 +271,62 @@ describe('direct APS rendering', () => { document.body.innerHTML = ''; }); - it('bootstraps the data renderer with a fragment-bound 128-bit nonce', () => { + it('loads the static route with a fragment-bound 128-bit nonce and opaque sandbox', () => { expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const slot = document.getElementById('fictional-slot')!; const iframe = slot.querySelector('iframe')!; const existing = slot.querySelector('span'); expect(existing).not.toBeNull(); - expect(iframe.src.startsWith(`${apsRendererBootstrapUrl()}#tsaps=`)).toBe(true); - expect(iframe.src).toMatch(/\?mode=data-bootstrap#tsaps=[A-Za-z0-9_-]{22}$/); + expect(iframe.src).toMatch(/\/integrations\/aps\/renderer#tsaps=[A-Za-z0-9_-]{22}$/); + expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); expect(iframe.getAttribute('sandbox')).not.toContain('allow-same-origin'); expect(iframe.srcdoc).toBe(''); - const { channel, sent } = advanceRendererToData(iframe); + const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); + iframe.dispatchEvent(new Event('load')); + expect(slot.querySelector('span')).not.toBeNull(); expect(iframe.style.display).toBe('none'); - expect(iframe.getAttribute('sandbox')).toContain('allow-same-origin'); - expect(sent).toEqual({ - nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), - publisherOrigin: window.location.origin, - renderer: descriptor(), - }); - - sendRendererMessage(channel, { - message: 'trusted-server/aps/renderer-ready', - nonce: `wrong-${sent.nonce}`, - }); - expect(slot.querySelector('span')).not.toBeNull(); + expect(postMessage).toHaveBeenCalledTimes(1); + expect(postMessage).toHaveBeenCalledWith( + { + nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), + renderer: descriptor(), + }, + '*' + ); + const message = postMessage.mock.calls[0][0] as { nonce: string }; window.dispatchEvent( new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, + data: { + message: 'trusted-server/aps/renderer-ready', + nonce: `wrong-${message.nonce}`, + }, source: iframe.contentWindow, }) ); expect(slot.querySelector('span')).not.toBeNull(); - expect(iframe.style.display).toBe('none'); - sendRendererMessage(channel, { - message: 'trusted-server/aps/renderer-ready', - nonce: sent.nonce, - }); - expect(iframe.getAttribute('sandbox')).toContain('allow-same-origin'); + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/renderer-ready', nonce: message.nonce }, + source: iframe.contentWindow, + }) + ); expect(slot.querySelector('span')).toBeNull(); expect(iframe.style.display).toBe(''); }); - it('accepts readiness only through the transferred renderer channel', () => { + it('rejects a ready message with the correct nonce from a foreign window', () => { expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const slot = document.getElementById('fictional-slot')!; const rendererFrame = slot.querySelector('iframe')!; - const { channel, sent } = advanceRendererToData(rendererFrame); + const postMessage = vi.spyOn(rendererFrame.contentWindow!, 'postMessage'); + rendererFrame.dispatchEvent(new Event('load')); + const sent = postMessage.mock.calls[0][0] as { nonce: string }; const foreignFrame = document.createElement('iframe'); document.body.appendChild(foreignFrame); @@ -406,6 +336,10 @@ describe('direct APS rendering', () => { source: foreignFrame.contentWindow, }) ); + + expect(slot.querySelector('span')).not.toBeNull(); + expect(rendererFrame.style.display).toBe('none'); + window.dispatchEvent( new MessageEvent('message', { data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, @@ -413,13 +347,6 @@ describe('direct APS rendering', () => { }) ); - expect(slot.querySelector('span')).not.toBeNull(); - expect(rendererFrame.style.display).toBe('none'); - - sendRendererMessage(channel, { - message: 'trusted-server/aps/renderer-ready', - nonce: sent.nonce, - }); expect(slot.querySelector('span')).toBeNull(); expect(rendererFrame.style.display).toBe(''); }); @@ -441,16 +368,6 @@ describe('direct APS rendering', () => { expect(document.querySelector('#fictional-slot iframe')).toBeNull(); }); - it('cancels a pending frame before another renderer replaces the slot', () => { - expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); - const container = document.getElementById('fictional-slot')!; - - cancelPendingApsRender(container); - - expect(container.querySelector('span')).not.toBeNull(); - expect(container.querySelector('iframe')).toBeNull(); - }); - it('removes an unacknowledged frame without clearing publisher content', () => { vi.useFakeTimers(); try { @@ -474,9 +391,9 @@ describe('direct APS rendering', () => { const baselineTimers = vi.getTimerCount(); expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const firstFrame = document.querySelector('#fictional-slot iframe')!; - const { channel: firstChannel, sent: firstSent } = advanceRendererToData( - firstFrame as HTMLIFrameElement - ); + const firstPostMessage = vi.spyOn(firstFrame.contentWindow!, 'postMessage'); + firstFrame.dispatchEvent(new Event('load')); + const firstSent = firstPostMessage.mock.calls[0][0] as { nonce: string }; const timersAfterFirst = vi.getTimerCount(); expect(timersAfterFirst).toBeGreaterThan(baselineTimers); @@ -484,21 +401,25 @@ describe('direct APS rendering', () => { const secondFrame = document.querySelector('#fictional-slot iframe')!; expect(firstFrame.isConnected).toBe(false); expect(vi.getTimerCount()).toBe(timersAfterFirst); - const { channel: secondChannel, sent } = advanceRendererToData( - secondFrame as HTMLIFrameElement - ); + const postMessage = vi.spyOn(secondFrame.contentWindow!, 'postMessage'); + secondFrame.dispatchEvent(new Event('load')); + const sent = postMessage.mock.calls[0][0] as { nonce: string }; - sendRendererMessage(firstChannel, { - message: 'trusted-server/aps/renderer-ready', - nonce: firstSent.nonce, - }); + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/renderer-ready', nonce: firstSent.nonce }, + source: firstFrame.contentWindow, + }) + ); expect(document.querySelector('#fictional-slot span')).not.toBeNull(); expect((secondFrame as HTMLIFrameElement).style.display).toBe('none'); - sendRendererMessage(secondChannel, { - message: 'trusted-server/aps/renderer-ready', - nonce: sent.nonce, - }); + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, + source: secondFrame.contentWindow, + }) + ); vi.advanceTimersByTime(10_000); expect(warnSpy).not.toHaveBeenCalled(); @@ -509,21 +430,19 @@ describe('direct APS rendering', () => { }); describe('Universal Creative APS source', () => { - it('uses the deployed dynamic renderer protocol to request a top-page mount', () => { - expect(APS_UNIVERSAL_CREATIVE_RENDERER_VERSION).toBeGreaterThanOrEqual(6); + it('uses the deployed dynamic renderer protocol and only creates the opaque route frame', () => { + expect(APS_UNIVERSAL_CREATIVE_RENDERER_VERSION).toBeGreaterThanOrEqual(4); expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('window.render=function'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.apsMountId'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.publisherOrigin'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('trusted-server/aps/mount-request'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('d&&d.apsRenderer'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('d&&d.rendererUrl'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(APS_RENDERER_DATA_URL); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(APS_RENDERER_SANDBOX); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.apsRenderer'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.rendererUrl'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain(APS_RENDERER_PATH); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain(APS_RENDERER_SANDBOX); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('allow-same-origin'); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('srcdoc'); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('document.write'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('creativeUrl'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('aaxResponse'); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('example-account-id'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(descriptor().creativeUrl); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(descriptor().aaxResponse); }); it('computes an absolute renderer URL from the publisher origin', () => { @@ -534,114 +453,31 @@ describe('Universal Creative APS source', () => { expect(apsRendererUrl('not an origin')).toBeUndefined(); }); - it('consumes a top-page mount capability once and preserves the controller frame', () => { - document.body.innerHTML = - '
'; - const container = document.getElementById('fictional-puc-slot')!; - const controller = container.querySelector('.puc-controller')!; - const requester = document.createElement('iframe'); - document.body.appendChild(requester); - const resultPost = vi.spyOn(requester.contentWindow!, 'postMessage'); - const mountId = registerApsUniversalCreativeMount(container, descriptor())!; - const requestNonce = 'ZYXWVUTSRQPONMLKJIHGFE'; - - const request = () => - window.dispatchEvent( - new MessageEvent('message', { - data: { - message: 'trusted-server/aps/mount-request', - mountId, - nonce: requestNonce, - }, - source: requester.contentWindow, - }) - ); - request(); - request(); - - const rendererFrame = container.querySelector( - 'iframe[data-ts-aps-renderer="true"]' - )!; - expect(rendererFrame).not.toBeNull(); - expect(container.querySelectorAll('iframe[data-ts-aps-renderer="true"]')).toHaveLength(1); - expect(controller.isConnected).toBe(true); - - const { channel, sent } = advanceRendererToData(rendererFrame); - sendRendererMessage(channel, { - message: 'trusted-server/aps/renderer-ready', - nonce: sent.nonce, - }); - - expect(controller.isConnected).toBe(true); - expect(controller.style.display).toBe('none'); - expect(rendererFrame.style.display).toBe(''); - expect(resultPost).toHaveBeenCalledWith( - { - message: 'trusted-server/aps/mount-result', - mountId, - nonce: requestNonce, - status: 'ready', - }, - '*' - ); - document.body.innerHTML = ''; - }); - - it('revokes an older mount capability when the same container is registered again', () => { - document.body.innerHTML = '
'; - const container = document.getElementById('fictional-refresh-slot')!; - const requester = document.createElement('iframe'); - document.body.appendChild(requester); - const oldMountId = registerApsUniversalCreativeMount(container, descriptor())!; - const newMountId = registerApsUniversalCreativeMount(container, descriptor())!; - const request = (mountId: string) => - window.dispatchEvent( - new MessageEvent('message', { - data: { - message: 'trusted-server/aps/mount-request', - mountId, - nonce: 'ABCDEFGHIJKLMNOPQRSTUV', - }, - source: requester.contentWindow, - }) - ); - - request(oldMountId); - expect(container.querySelector('iframe[data-ts-aps-renderer="true"]')).toBeNull(); - request(newMountId); - const rendererFrame = container.querySelector( - 'iframe[data-ts-aps-renderer="true"]' - ); - expect(rendererFrame).not.toBeNull(); - rendererFrame!.dispatchEvent(new Event('error')); - document.body.innerHTML = ''; - }); - - it('resolves only after the top page acknowledges its one-shot mount request', async () => { + it('creates the opaque route frame and resolves only after the bound acknowledgement', async () => { const dynamicWindow = window as unknown as { - render?: (data: Record) => Promise; + render?: (data: Record, helper: unknown, target: Window) => Promise; }; - const postMessage = vi.spyOn(window.top, 'postMessage'); window.eval(APS_UNIVERSAL_CREATIVE_RENDERER); try { - const mountId = 'ABCDEFGHIJKLMNOPQRSTUV'; - const rendered = dynamicWindow.render!({ - apsMountId: mountId, - publisherOrigin: window.location.origin, - }); - expect(document.body.querySelector('iframe')).toBeNull(); - expect(postMessage).toHaveBeenCalledTimes(1); - const sent = postMessage.mock.calls[0][0] as { - message: string; - mountId: string; - nonce: string; - }; - expect(sent).toEqual({ - message: 'trusted-server/aps/mount-request', - mountId, - nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), - }); + const renderer = descriptor(); + const rendered = dynamicWindow.render!( + { + apsRenderer: renderer, + rendererUrl: apsRendererUrl(), + }, + undefined, + window + ); + const iframe = document.body.querySelector('iframe')!; + expect(iframe.src).toMatch(/\/integrations\/aps\/renderer#tsaps=[A-Za-z0-9_-]{22}$/); + expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); + expect(iframe.getAttribute('sandbox')).not.toContain('allow-same-origin'); + + const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); + iframe.dispatchEvent(new Event('load')); + const sent = postMessage.mock.calls[0][0] as { nonce: string; renderer: ApsRendererV1 }; + expect(sent.renderer).toEqual(renderer); let settled = false; void rendered.then(() => { @@ -652,18 +488,12 @@ describe('Universal Creative APS source', () => { window.dispatchEvent( new MessageEvent('message', { - data: { - message: 'trusted-server/aps/mount-result', - mountId, - nonce: sent.nonce, - status: 'ready', - }, - source: window.top, + data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, + source: iframe.contentWindow, }) ); await expect(rendered).resolves.toBeUndefined(); } finally { - postMessage.mockRestore(); delete dynamicWindow.render; document.body.innerHTML = ''; } diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 4a95d04e8..63caff94e 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -3176,12 +3176,11 @@ describe('installTsRenderBridge', () => { expect(Object.keys(response).sort()).toEqual( [ 'adId', - 'apsMountId', 'apsRenderer', 'height', 'message', - 'publisherOrigin', 'renderer', + 'rendererUrl', 'rendererVersion', 'width', ].sort() @@ -3190,9 +3189,8 @@ describe('installTsRenderBridge', () => { message: 'Prebid Response', adId: renderer.bidId, renderer: expect.stringContaining('window.render=function'), - rendererVersion: 6, - apsMountId: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), - publisherOrigin: window.location.origin, + rendererVersion: 4, + rendererUrl: new URL('/integrations/aps/renderer', window.location.origin).href, apsRenderer: renderer, width: 300, height: 250, @@ -3200,9 +3198,35 @@ describe('installTsRenderBridge', () => { expect(String(response.renderer)).not.toContain(renderer.accountId); expect(String(response.renderer)).not.toContain(renderer.aaxResponse); - expect(String(response.renderer)).toContain('d&&d.apsMountId'); - expect(String(response.renderer)).not.toContain(renderer.creativeUrl); - expect(String(response.renderer)).not.toContain(renderer.aaxResponse); + // Universal Creative's dynamic-renderer path evaluates the returned static + // source and calls window.render(response, helper, targetWindow). Consume + // the exact bridge response through that deployed protocol shape. + const dynamicWindow = window as unknown as { + render?: (data: Record, helper: unknown, target: Window) => Promise; + }; + window.eval(String(response.renderer)); + try { + const rendered = dynamicWindow.render!(response, undefined, window); + const outerFrame = document.querySelector( + 'iframe[src*="/integrations/aps/renderer#tsaps="]' + )!; + expect(outerFrame).not.toBeNull(); + expect(outerFrame.getAttribute('sandbox')).not.toContain('allow-same-origin'); + + const rendererPost = vi.spyOn(outerFrame.contentWindow!, 'postMessage'); + outerFrame.dispatchEvent(new Event('load')); + const sent = rendererPost.mock.calls[0][0] as { nonce: string }; + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, + source: outerFrame.contentWindow, + }) + ); + await expect(rendered).resolves.toBeUndefined(); + outerFrame.remove(); + } finally { + delete dynamicWindow.render; + } beaconSpy.mockRestore(); }); diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index 3ab6e904b..658f5d5bc 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -150,27 +150,30 @@ Trusted Server does not insert APS creative markup into the publisher document. Seats, `impid`, markup, notifications, user-sync data, sibling bids, losing seats, and unknown fields are not exposed. The browser decodes this envelope and cross-checks the ID, dimensions, URL, and tag type before any DOM mutation or message suppression. -Both rendering paths use a publisher-origin bootstrap followed by two nested `data:` documents. TSJS first loads `GET /integrations/aps/renderer?mode=data-bootstrap` with a sandbox that omits `allow-same-origin`. The bootstrap is therefore opaque and cannot read or modify the publisher document. After a nonce-bound readiness message, TSJS adds `allow-same-origin` and asks the bootstrap to navigate itself to a per-impression `data:` container. The container then creates the static inner `data:` renderer under the same permanent sandbox. +Both rendering paths use `GET /integrations/aps/renderer`, a static Trusted Server document with its own restrictive CSP. The document initializes the account-keyed APS queue and then loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. -Both data documents are naturally opaque even with `allow-same-origin`, so the publisher cannot access them. Keeping that token on both frames also avoids WebKit propagating an opaque origin to the HTTPS creative: the creative and its same-origin descendants retain their real origin in Chromium, Firefox, and WebKit. +The outer iframe uses these sandbox permissions: -The outer container's CSP allows child frames from `data:` and only the fully validated creative URL's exact origin. That policy is inherited by the inner data renderer and intersects with the renderer's own CSP. It permits the expected creative frame but blocks the inner renderer from navigating itself to the publisher origin before a request is made. This exact-origin boundary intentionally blocks an immediate creative-frame redirect or same-frame navigation to a different origin; validate real APS inventory for intermediate or redirect origins before rollout. User-activated top navigation and popups remain governed by the sandbox tokens. - -A network-loaded HTTPS creative does not inherit its ancestor's CSP and can create further frames. To prevent it from using a publisher-origin descendant plus an executable publisher gadget to regain access to the top page, enabling APS appends a separate `Content-Security-Policy: frame-ancestors 'self'` policy to every Trusted Server response. Browsers require every ancestor to match, so publisher documents cannot load below the opaque container or third-party creative. This secure default also prevents the APS-enabled publisher from being embedded cross-origin; publishers that require trusted external embedders need a separately reviewed ancestor-allowlist feature before enabling APS. - -Trusted Server generates independent 128-bit nonces for the container and renderer. Once the inner renderer is ready, the container transfers a dedicated `MessagePort` between trusted top-page TSJS and the inner renderer. The complete descriptor travels only over that port; the inner renderer revalidates it, rejects the publisher origin, and requires the creative origin to equal the origin bound in the container CSP. The container URL contains only that exact origin, static renderer source, and nonces—never the response envelope, bid ID, price, or creative URL path. +```text +allow-forms +allow-pointer-lock +allow-popups +allow-popups-to-escape-sandbox +allow-scripts +allow-top-navigation-by-user-activation +``` -The inner document initializes the account-keyed APS queue and loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. A `renderer-ready` acknowledgement still means that Amazon's runner loaded, not that the final creative painted. Existing slot content remains until that acknowledgement. The query-free `GET /integrations/aps/renderer` response remains available with its original stricter CSP and legacy message shape for cached clients and rollback. +It deliberately omits `allow-same-origin`, so APS and bidder execution remains below an opaque-origin boundary. The renderer response repeats these restrictions with a CSP `sandbox` directive, preventing another embedding path from restoring publisher-origin execution by omitting the iframe attribute. Trusted Server generates a fresh 128-bit nonce, binds it in the iframe URL fragment before navigation, and requires the same one-time nonce in the parent message and renderer acknowledgement. Existing slot content is retained until the static renderer has accepted the descriptor and loaded the fixed runner. ### Direct `/auction` -The TSJS auction client validates the typed renderer descriptor and mounts the nested data renderer in the winning slot. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. +The TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. ### GAM and Universal Creative -For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program with a one-shot mount capability. That program asks trusted top-page TSJS to mount the nested data renderer as a sibling of the Universal Creative iframe, outside inherited GAM and Universal Creative sandbox restrictions. +For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program that creates the same opaque renderer iframe. -For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes both the renderer and mount capabilities once, and passes the APS bid ID separately to the Amazon runner. +For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes the capability once, and passes the APS bid ID separately to the Amazon runner. These paths do not fetch PBS Cache, fire generic APS win/billing beacons, or call `apstag.setDisplayBids()` for the Trusted Server winner. Publisher-owned native APS objects are otherwise left untouched. @@ -179,10 +182,10 @@ These paths do not fetch PBS Cache, fire generic APS win/billing beacons, or cal The publisher policy must permit the same-origin renderer route, for example: ```text -frame-src 'self' data: +frame-src 'self' ``` -The `data:` source is required for the bootstrap's self-navigation to the opaque container. APS also adds `frame-ancestors 'self'` as an independent response policy; do not override or remove that policy. Do not weaken or bypass the initial bootstrap sandbox. TSJS adds `allow-same-origin` only when navigating away from the publisher-origin bootstrap; both final data documents remain naturally opaque. The bootstrap response supplies the resource CSP for the fixed runner and HTTPS creative resources, while the container narrows `frame-src` to the validated creative origin. The same-origin bootstrap route inherits the publisher page scheme; use HTTPS in production. APS endpoints and third-party creative URLs always require HTTPS. +Do not add `allow-same-origin` to the outer renderer sandbox. The renderer endpoint supplies its own CSP for the fixed runner and HTTPS creative resources. The same-origin renderer route inherits the publisher page scheme; use HTTPS in production. APS endpoints and third-party creative URLs always require HTTPS. Before enabling script creatives, verify under the publisher's actual CSP that both iframe and script-tag creatives: @@ -232,8 +235,8 @@ Use fictional values in source-controlled configuration and fixtures. Supply con ### Winner targets but does not render -- Confirm `GET /integrations/aps/renderer?mode=data-bootstrap` returns HTML with its bootstrap CSP and `Referrer-Policy: no-referrer`. -- Confirm publisher CSP permits `frame-src 'self' data:`. +- Confirm `GET /integrations/aps/renderer` returns HTML with its CSP and `Referrer-Policy: no-referrer`. +- Confirm publisher CSP permits `frame-src 'self'`. - Confirm the GAM creative uses the supported Prebid Universal Creative bridge and the winning `hb_adid`. - For client-side `trustedServer` adapter auctions, confirm Prebid's `bidResponse` contains a generated `adId` and that the corresponding capability appears briefly in `window.tsjs.apsPrebidRenderers` before rendering. - Ensure no native APS path is trying to handle the same cohort. From 3953cd6b31217d986973c9c525ebfa5da3c0fe33 Mon Sep 17 00:00:00 2001 From: Christian Date: Tue, 18 Aug 2026 11:10:34 -0500 Subject: [PATCH 330/395] Reapply "Merge remote-tracking branch 'origin/pr/1033' into rc/202608" This reverts commit bf9cfdd132652174d190dd0cdc785e0f136e2ede. --- .../src/integrations/aps.rs | 125 ++-- .../src/response_privacy.rs | 64 ++ ...prebid-universal-creative-1.17.2-banner.js | 4 + .../browser/tests/shared/aps-renderer.spec.ts | 706 +++++++++++++++++- .../trusted-server-js/lib/src/core/request.ts | 3 +- .../lib/src/integrations/aps/render.ts | 401 ++++++++-- .../integrations/aps/renderer-bootstrap.html | 41 + .../integrations/aps/renderer-container.html | 71 ++ .../lib/src/integrations/aps/renderer.html | 228 ++++++ .../lib/src/integrations/gpt/index.ts | 36 +- .../lib/test/core/request.test.ts | 48 +- .../lib/test/integrations/aps/render.test.ts | 334 +++++++-- .../lib/test/integrations/gpt/ad_init.test.ts | 40 +- docs/guide/integrations/aps.md | 33 +- 14 files changed, 1820 insertions(+), 314 deletions(-) create mode 100644 crates/trusted-server-integration-tests/browser/fixtures/prebid-universal-creative-1.17.2-banner.js create mode 100644 crates/trusted-server-js/lib/src/integrations/aps/renderer-bootstrap.html create mode 100644 crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html create mode 100644 crates/trusted-server-js/lib/src/integrations/aps/renderer.html diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 4fed9278d..6121d6261 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -36,6 +36,7 @@ use crate::settings::{IntegrationConfig, Settings}; const APS_INTEGRATION_ID: &str = "aps"; const APS_RENDERER_ROUTE: &str = "/integrations/aps/renderer"; +const APS_RENDERER_BOOTSTRAP_QUERY: &str = "mode=data-bootstrap"; const DEFAULT_CURRENCY: &str = "USD"; const APS_SDK_SOURCE: &str = "prebid"; const APS_SDK_VERSION: &str = "2.2.0"; @@ -47,72 +48,12 @@ const MAX_LANGUAGE_BYTES: usize = 8; const MAX_PAGE_URL_BYTES: usize = 8192; const MAX_RENDER_ENVELOPE_BYTES: usize = 256 * 1024; const APS_RENDERER_CSP: &str = "default-src 'none'; sandbox allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation; script-src 'unsafe-inline' https:; connect-src https:; frame-src https:; img-src https: data:; media-src https: blob:; style-src 'unsafe-inline' https:; font-src https: data:;"; +const APS_RENDERER_BOOTSTRAP_CSP: &str = "default-src 'none'; sandbox allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-top-navigation-by-user-activation; script-src 'unsafe-inline' https:; connect-src https:; frame-src https: data:; img-src https: data:; media-src https: blob:; style-src 'unsafe-inline' https:; font-src https: data:;"; -const APS_RENDERER_DOCUMENT: &str = r#" - - -"#; +const APS_RENDERER_DOCUMENT: &str = + include_str!("../../../trusted-server-js/lib/src/integrations/aps/renderer.html"); +const APS_RENDERER_BOOTSTRAP_DOCUMENT: &str = + include_str!("../../../trusted-server-js/lib/src/integrations/aps/renderer-bootstrap.html"); /// Configuration for the APS `OpenRTB` integration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] @@ -1211,13 +1152,28 @@ impl IntegrationProxy for ApsRendererIntegration { message: "Failed to build APS not-found response".to_string(), }); } + let (renderer_document, renderer_csp) = match request.uri().query() { + None => (APS_RENDERER_DOCUMENT, APS_RENDERER_CSP), + Some(APS_RENDERER_BOOTSTRAP_QUERY) => { + (APS_RENDERER_BOOTSTRAP_DOCUMENT, APS_RENDERER_BOOTSTRAP_CSP) + } + Some(_) => { + return http::Response::builder() + .status(StatusCode::NOT_FOUND) + .body(EdgeBody::from("Not Found")) + .change_context(TrustedServerError::Integration { + integration: APS_INTEGRATION_ID.to_string(), + message: "Failed to build APS not-found response".to_string(), + }); + } + }; http::Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "text/html; charset=utf-8") .header("x-content-type-options", "nosniff") .header("referrer-policy", "no-referrer") - .header(header::CONTENT_SECURITY_POLICY, APS_RENDERER_CSP) - .body(EdgeBody::from(APS_RENDERER_DOCUMENT)) + .header(header::CONTENT_SECURITY_POLICY, renderer_csp) + .body(EdgeBody::from(renderer_document)) .change_context(TrustedServerError::Integration { integration: APS_INTEGRATION_ID.to_string(), message: "Failed to build APS renderer response".to_string(), @@ -2318,7 +2274,7 @@ mod tests { } #[test] - fn registers_and_serves_only_static_renderer_route() { + fn registers_and_serves_static_renderer_and_data_bootstrap_modes() { let integration = ApsRendererIntegration; let routes = integration.routes(); assert_eq!(routes.len(), 1, "should register one route"); @@ -2347,6 +2303,25 @@ mod tests { APS_RENDERER_CSP ); + let bootstrap = http::Request::builder() + .method(Method::GET) + .uri(format!( + "{APS_RENDERER_ROUTE}?{APS_RENDERER_BOOTSTRAP_QUERY}" + )) + .body(EdgeBody::empty()) + .expect("should build renderer bootstrap request"); + let response = + futures::executor::block_on(integration.handle(&settings, &services, bootstrap)) + .expect("should serve renderer bootstrap"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_SECURITY_POLICY], + APS_RENDERER_BOOTSTRAP_CSP + ); + assert!(APS_RENDERER_BOOTSTRAP_CSP.contains("allow-same-origin")); + assert!(APS_RENDERER_BOOTSTRAP_CSP.contains("frame-src https: data:")); + assert!(APS_RENDERER_BOOTSTRAP_DOCUMENT.contains("trusted-server/aps/bootstrap-navigate")); + let post = http::Request::builder() .method(Method::POST) .uri(APS_RENDERER_ROUTE) @@ -2374,6 +2349,7 @@ mod tests { assert_eq!(registration.integration_id, APS_INTEGRATION_ID); assert_eq!(registration.proxies.len(), 1); + assert!(registration.request_filters.is_empty()); assert!(registration.js_disabled); } @@ -2425,12 +2401,15 @@ mod tests { #[test] fn renderer_document_is_static_and_nonce_bound() { assert!(APS_RENDERER_DOCUMENT.contains("^#tsaps=")); - assert!(APS_RENDERER_DOCUMENT.contains("event.source!==parent")); - assert!(APS_RENDERER_DOCUMENT.contains("message.nonce!==expected")); + assert!(APS_RENDERER_DOCUMENT.contains("event.source !== parent")); + assert!(APS_RENDERER_DOCUMENT.contains("message.nonce !== expected")); + assert!(APS_RENDERER_DOCUMENT.contains("['nonce', 'publisherOrigin', 'renderer']")); + assert!(APS_RENDERER_DOCUMENT.contains("['nonce', 'renderer']")); + assert!(APS_RENDERER_DOCUMENT.contains("url.origin === publisherOrigin")); assert!(APS_RENDERER_DOCUMENT.contains("prebid/creative/render")); assert!(APS_RENDERER_DOCUMENT.contains("window._aps instanceof Map")); - assert!(APS_RENDERER_DOCUMENT.contains("store:new Map([['listeners',new Map()]])")); - assert!(APS_RENDERER_DOCUMENT.contains("account.queue.push(new CustomEvent")); + assert!(APS_RENDERER_DOCUMENT.contains("store: new Map([['listeners', new Map()]])")); + assert!(APS_RENDERER_DOCUMENT.contains("account.queue.push(")); assert!( APS_RENDERER_DOCUMENT.contains("trusted-server/aps/renderer-ready") && APS_RENDERER_DOCUMENT.contains("trusted-server/aps/renderer-failed") @@ -2442,7 +2421,7 @@ mod tests { ); assert!(!APS_RENDERER_DOCUMENT.contains(" diff --git a/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html b/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html new file mode 100644 index 000000000..6ef871c9b --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html @@ -0,0 +1,71 @@ + + + + + + + + diff --git a/crates/trusted-server-js/lib/src/integrations/aps/renderer.html b/crates/trusted-server-js/lib/src/integrations/aps/renderer.html new file mode 100644 index 000000000..b7ddac1c0 --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/aps/renderer.html @@ -0,0 +1,228 @@ + + + + + diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 52a6ecb70..ef6a3204c 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -10,9 +10,9 @@ import type { import { APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - apsRendererUrl, consumeApsPrebidRenderer, getApsPrebidRenderer, + registerApsUniversalCreativeMount, validateApsRenderer, } from '../aps/render'; @@ -225,13 +225,14 @@ function slotIdForMessageSource(source: MessageEventSource | null): string | und ?.id; } -function messageSourceBelongsToAdUnit( +function slotRootForMessageSource( source: MessageEventSource | null, - adUnitCode: string -): boolean { - return source - ? sourceIsInSlotRoots(source, candidateSlotRootsForConfiguredDivId(adUnitCode)) - : false; + divId: string +): HTMLElement | undefined { + if (!source) return undefined; + return candidateSlotRootsForConfiguredDivId(divId).find((root) => + sourceIsInSlotRoots(source, [root]) + ); } function clearTargetingKeys(slot: GoogleTagSlot, keys: Iterable): void { @@ -1700,13 +1701,15 @@ export function installTsRenderBridge(): void { // Prebid handles ad IDs globally and would otherwise answer a request from // an unrelated iframe when this slot-bound capability rejects it. e.stopImmediatePropagation(); - if (!messageSourceBelongsToAdUnit(e.source, prebidRendererEntry.adUnitCode)) return; + const mountContainer = slotRootForMessageSource(e.source, prebidRendererEntry.adUnitCode); + if (!mountContainer) return; const renderer = validateApsRenderer(prebidRendererEntry.renderer); - const rendererUrl = apsRendererUrl(); - if (!renderer || !rendererUrl) return; + if (!renderer) return; if (!hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; if (!consumeApsPrebidRenderer(adId, prebidRendererEntry)) return; recordConsumedPrebidApsId(consumedPrebidApsIds, adId, prebidRendererEntry.expiresAt); + const apsMountId = registerApsUniversalCreativeMount(mountContainer, renderer); + if (!apsMountId) return; port.postMessage( JSON.stringify({ @@ -1714,7 +1717,8 @@ export function installTsRenderBridge(): void { adId, renderer: APS_UNIVERSAL_CREATIVE_RENDERER, rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - rendererUrl, + apsMountId, + publisherOrigin: window.location.origin, apsRenderer: renderer, width: renderer.width, height: renderer.height, @@ -1753,8 +1757,11 @@ export function installTsRenderBridge(): void { e.stopImmediatePropagation(); if (consumedServerApsBySlot.get(slotId) === adId) return; const renderer = validateApsRenderer(matchedBid.renderer); - const rendererUrl = apsRendererUrl(); - if (!renderer || !rendererUrl) return; + const configuredSlot = window.tsjs?.adSlots?.find((slot) => slot.id === slotId); + const mountContainer = slotRootForMessageSource(e.source, configuredSlot?.div_id ?? slotId); + if (!renderer || !mountContainer) return; + const apsMountId = registerApsUniversalCreativeMount(mountContainer, renderer); + if (!apsMountId) return; consumedServerApsBySlot.set(slotId, adId); port.postMessage( JSON.stringify({ @@ -1762,7 +1769,8 @@ export function installTsRenderBridge(): void { adId, renderer: APS_UNIVERSAL_CREATIVE_RENDERER, rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - rendererUrl, + apsMountId, + publisherOrigin: window.location.origin, apsRenderer: renderer, width: renderer.width, height: renderer.height, diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index dc17c9e87..ba2dfb274 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -69,7 +69,7 @@ describe('request.requestAds', () => { ); }); - it('dispatches a valid APS descriptor to the opaque static renderer route', async () => { + it('dispatches a valid APS descriptor through the opaque data renderer bootstrap', async () => { const apsBid = envelope.seatbid[0].bid[0]; const renderer = { type: 'aps', @@ -117,21 +117,55 @@ describe('request.requestAds', () => { const iframe = document.querySelector('#slot1 iframe') as HTMLIFrameElement | null; expect(iframe).not.toBeNull(); - expect(iframe!.src).toContain('/integrations/aps/renderer#tsaps='); + expect(iframe!.src).toContain('/integrations/aps/renderer?mode=data-bootstrap#tsaps='); expect(iframe!.srcdoc).toBe(''); expect(iframe!.getAttribute('sandbox')).not.toContain('allow-same-origin'); expect(document.querySelector('#slot1 span')).not.toBeNull(); const postMessage = vi.spyOn(iframe!.contentWindow!, 'postMessage'); - iframe!.dispatchEvent(new Event('load')); + const nonce = new URL(iframe!.src).hash.replace('#tsaps=', ''); + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/bootstrap-ready', nonce }, + source: iframe!.contentWindow, + }) + ); + const navigate = postMessage.mock.calls[0][0] as { rendererUrl: string }; + const containerDocument = decodeURIComponent( + navigate.rendererUrl + .slice('data:text/html;charset=utf-8,'.length) + .replace(/#tsaps=[A-Za-z0-9_-]{22}$/, '') + ); + const innerNonce = containerDocument.match( + /data:text\/html;charset=utf-8,.+#tsaps=([A-Za-z0-9_-]{22})/ + )?.[1]; + expect(innerNonce).toMatch(/^[A-Za-z0-9_-]{22}$/); + + const channel = { + close: vi.fn(), + onmessage: null, + postMessage: vi.fn(), + start: vi.fn(), + } as unknown as MessagePort; + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/container-ready', nonce }, + source: iframe!.contentWindow, + ports: [channel], + }) + ); + channel.onmessage?.( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/channel-ready', nonce: innerNonce }, + }) + ); expect(document.querySelector('#slot1 span')).not.toBeNull(); - expect(postMessage).toHaveBeenCalledWith(expect.objectContaining({ renderer }), '*'); + expect(channel.postMessage).toHaveBeenCalledWith(expect.objectContaining({ renderer })); - const message = postMessage.mock.calls[0][0] as { nonce: string }; - window.dispatchEvent( + const message = vi.mocked(channel.postMessage).mock.calls[0][0] as { nonce: string }; + channel.onmessage?.( new MessageEvent('message', { data: { message: 'trusted-server/aps/renderer-ready', nonce: message.nonce }, - source: iframe!.contentWindow, }) ); expect(document.querySelector('#slot1 span')).toBeNull(); diff --git a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts index eae60c90e..c9808b8ac 100644 --- a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts @@ -4,14 +4,18 @@ import envelope from '../../fixtures/aps-renderer-v1.json'; import type { ApsRendererV1 } from '../../../src/core/types'; import { log } from '../../../src/core/log'; import { + APS_RENDERER_DATA_URL, APS_RENDERER_PATH, APS_RENDERER_SANDBOX, APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, + apsRendererBootstrapUrl, apsRendererUrl, + cancelPendingApsRender, getApsPrebidRenderer, parseApsRendererDescriptor, registerApsPrebidRenderer, + registerApsUniversalCreativeMount, renderApsCreative, validateApsRenderer, } from '../../../src/integrations/aps/render'; @@ -50,6 +54,76 @@ function descriptor(overrides: Partial = {}): ApsRendererV1 { }; } +type FakeRendererChannel = MessagePort & { + close: ReturnType; + postMessage: ReturnType; + start: ReturnType; +}; + +function sendRendererMessage(channel: FakeRendererChannel, data: Record): void { + channel.onmessage?.(new MessageEvent('message', { data })); +} + +function advanceRendererToData(iframe: HTMLIFrameElement) { + const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); + const nonce = new URL(iframe.src).hash.replace('#tsaps=', ''); + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/bootstrap-ready', nonce }, + source: iframe.contentWindow, + }) + ); + expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); + const navigate = postMessage.mock.calls[0][0] as { + message: string; + nonce: string; + rendererUrl: string; + }; + expect(navigate.message).toBe('trusted-server/aps/bootstrap-navigate'); + expect(navigate.nonce).toBe(nonce); + expect(navigate.rendererUrl).toMatch( + /^data:text\/html;charset=utf-8,.+#tsaps=[A-Za-z0-9_-]{22}$/ + ); + + const encodedDocument = navigate.rendererUrl + .slice('data:text/html;charset=utf-8,'.length) + .replace(/#tsaps=[A-Za-z0-9_-]{22}$/, ''); + const containerDocument = decodeURIComponent(encodedDocument); + const innerNonce = containerDocument.match( + /data:text\/html;charset=utf-8,.+#tsaps=([A-Za-z0-9_-]{22})/ + )?.[1]; + expect(innerNonce).toMatch(/^[A-Za-z0-9_-]{22}$/); + expect(innerNonce).not.toBe(nonce); + expect(containerDocument).toContain('frame-src data: https://creative.example'); + expect(containerDocument).not.toContain(descriptor().bidId); + expect(containerDocument).not.toContain(descriptor().aaxResponse); + + const channel = { + close: vi.fn(), + onmessage: null, + postMessage: vi.fn(), + start: vi.fn(), + } as unknown as FakeRendererChannel; + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/container-ready', nonce }, + source: iframe.contentWindow, + ports: [channel], + }) + ); + expect(channel.start).toHaveBeenCalledOnce(); + sendRendererMessage(channel, { + message: 'trusted-server/aps/channel-ready', + nonce: innerNonce, + }); + const sent = channel.postMessage.mock.calls[0][0] as { + nonce: string; + publisherOrigin: string; + renderer: ApsRendererV1; + }; + return { channel, innerNonce: innerNonce!, postMessage, sent }; +} + describe('APS renderer validation', () => { it('consumes the shared fictional golden envelope and supports an omitted creative ID', () => { const withCreativeId = descriptor(); @@ -271,62 +345,58 @@ describe('direct APS rendering', () => { document.body.innerHTML = ''; }); - it('loads the static route with a fragment-bound 128-bit nonce and opaque sandbox', () => { + it('bootstraps the data renderer with a fragment-bound 128-bit nonce', () => { expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const slot = document.getElementById('fictional-slot')!; const iframe = slot.querySelector('iframe')!; const existing = slot.querySelector('span'); expect(existing).not.toBeNull(); - expect(iframe.src).toMatch(/\/integrations\/aps\/renderer#tsaps=[A-Za-z0-9_-]{22}$/); - expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); + expect(iframe.src.startsWith(`${apsRendererBootstrapUrl()}#tsaps=`)).toBe(true); + expect(iframe.src).toMatch(/\?mode=data-bootstrap#tsaps=[A-Za-z0-9_-]{22}$/); expect(iframe.getAttribute('sandbox')).not.toContain('allow-same-origin'); expect(iframe.srcdoc).toBe(''); - const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); - iframe.dispatchEvent(new Event('load')); - + const { channel, sent } = advanceRendererToData(iframe); expect(slot.querySelector('span')).not.toBeNull(); expect(iframe.style.display).toBe('none'); - expect(postMessage).toHaveBeenCalledTimes(1); - expect(postMessage).toHaveBeenCalledWith( - { - nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), - renderer: descriptor(), - }, - '*' - ); + expect(iframe.getAttribute('sandbox')).toContain('allow-same-origin'); + expect(sent).toEqual({ + nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), + publisherOrigin: window.location.origin, + renderer: descriptor(), + }); + + sendRendererMessage(channel, { + message: 'trusted-server/aps/renderer-ready', + nonce: `wrong-${sent.nonce}`, + }); + expect(slot.querySelector('span')).not.toBeNull(); - const message = postMessage.mock.calls[0][0] as { nonce: string }; window.dispatchEvent( new MessageEvent('message', { - data: { - message: 'trusted-server/aps/renderer-ready', - nonce: `wrong-${message.nonce}`, - }, + data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, source: iframe.contentWindow, }) ); expect(slot.querySelector('span')).not.toBeNull(); + expect(iframe.style.display).toBe('none'); - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: message.nonce }, - source: iframe.contentWindow, - }) - ); + sendRendererMessage(channel, { + message: 'trusted-server/aps/renderer-ready', + nonce: sent.nonce, + }); + expect(iframe.getAttribute('sandbox')).toContain('allow-same-origin'); expect(slot.querySelector('span')).toBeNull(); expect(iframe.style.display).toBe(''); }); - it('rejects a ready message with the correct nonce from a foreign window', () => { + it('accepts readiness only through the transferred renderer channel', () => { expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const slot = document.getElementById('fictional-slot')!; const rendererFrame = slot.querySelector('iframe')!; - const postMessage = vi.spyOn(rendererFrame.contentWindow!, 'postMessage'); - rendererFrame.dispatchEvent(new Event('load')); - const sent = postMessage.mock.calls[0][0] as { nonce: string }; + const { channel, sent } = advanceRendererToData(rendererFrame); const foreignFrame = document.createElement('iframe'); document.body.appendChild(foreignFrame); @@ -336,10 +406,6 @@ describe('direct APS rendering', () => { source: foreignFrame.contentWindow, }) ); - - expect(slot.querySelector('span')).not.toBeNull(); - expect(rendererFrame.style.display).toBe('none'); - window.dispatchEvent( new MessageEvent('message', { data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, @@ -347,6 +413,13 @@ describe('direct APS rendering', () => { }) ); + expect(slot.querySelector('span')).not.toBeNull(); + expect(rendererFrame.style.display).toBe('none'); + + sendRendererMessage(channel, { + message: 'trusted-server/aps/renderer-ready', + nonce: sent.nonce, + }); expect(slot.querySelector('span')).toBeNull(); expect(rendererFrame.style.display).toBe(''); }); @@ -368,6 +441,16 @@ describe('direct APS rendering', () => { expect(document.querySelector('#fictional-slot iframe')).toBeNull(); }); + it('cancels a pending frame before another renderer replaces the slot', () => { + expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); + const container = document.getElementById('fictional-slot')!; + + cancelPendingApsRender(container); + + expect(container.querySelector('span')).not.toBeNull(); + expect(container.querySelector('iframe')).toBeNull(); + }); + it('removes an unacknowledged frame without clearing publisher content', () => { vi.useFakeTimers(); try { @@ -391,9 +474,9 @@ describe('direct APS rendering', () => { const baselineTimers = vi.getTimerCount(); expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const firstFrame = document.querySelector('#fictional-slot iframe')!; - const firstPostMessage = vi.spyOn(firstFrame.contentWindow!, 'postMessage'); - firstFrame.dispatchEvent(new Event('load')); - const firstSent = firstPostMessage.mock.calls[0][0] as { nonce: string }; + const { channel: firstChannel, sent: firstSent } = advanceRendererToData( + firstFrame as HTMLIFrameElement + ); const timersAfterFirst = vi.getTimerCount(); expect(timersAfterFirst).toBeGreaterThan(baselineTimers); @@ -401,25 +484,21 @@ describe('direct APS rendering', () => { const secondFrame = document.querySelector('#fictional-slot iframe')!; expect(firstFrame.isConnected).toBe(false); expect(vi.getTimerCount()).toBe(timersAfterFirst); - const postMessage = vi.spyOn(secondFrame.contentWindow!, 'postMessage'); - secondFrame.dispatchEvent(new Event('load')); - const sent = postMessage.mock.calls[0][0] as { nonce: string }; - - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: firstSent.nonce }, - source: firstFrame.contentWindow, - }) + const { channel: secondChannel, sent } = advanceRendererToData( + secondFrame as HTMLIFrameElement ); + + sendRendererMessage(firstChannel, { + message: 'trusted-server/aps/renderer-ready', + nonce: firstSent.nonce, + }); expect(document.querySelector('#fictional-slot span')).not.toBeNull(); expect((secondFrame as HTMLIFrameElement).style.display).toBe('none'); - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, - source: secondFrame.contentWindow, - }) - ); + sendRendererMessage(secondChannel, { + message: 'trusted-server/aps/renderer-ready', + nonce: sent.nonce, + }); vi.advanceTimersByTime(10_000); expect(warnSpy).not.toHaveBeenCalled(); @@ -430,19 +509,21 @@ describe('direct APS rendering', () => { }); describe('Universal Creative APS source', () => { - it('uses the deployed dynamic renderer protocol and only creates the opaque route frame', () => { - expect(APS_UNIVERSAL_CREATIVE_RENDERER_VERSION).toBeGreaterThanOrEqual(4); + it('uses the deployed dynamic renderer protocol to request a top-page mount', () => { + expect(APS_UNIVERSAL_CREATIVE_RENDERER_VERSION).toBeGreaterThanOrEqual(6); expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('window.render=function'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.apsRenderer'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.rendererUrl'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain(APS_RENDERER_PATH); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain(APS_RENDERER_SANDBOX); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('allow-same-origin'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.apsMountId'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.publisherOrigin'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('trusted-server/aps/mount-request'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('d&&d.apsRenderer'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('d&&d.rendererUrl'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(APS_RENDERER_DATA_URL); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(APS_RENDERER_SANDBOX); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('srcdoc'); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('document.write'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('creativeUrl'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('aaxResponse'); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('example-account-id'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(descriptor().creativeUrl); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(descriptor().aaxResponse); }); it('computes an absolute renderer URL from the publisher origin', () => { @@ -453,31 +534,114 @@ describe('Universal Creative APS source', () => { expect(apsRendererUrl('not an origin')).toBeUndefined(); }); - it('creates the opaque route frame and resolves only after the bound acknowledgement', async () => { + it('consumes a top-page mount capability once and preserves the controller frame', () => { + document.body.innerHTML = + '
'; + const container = document.getElementById('fictional-puc-slot')!; + const controller = container.querySelector('.puc-controller')!; + const requester = document.createElement('iframe'); + document.body.appendChild(requester); + const resultPost = vi.spyOn(requester.contentWindow!, 'postMessage'); + const mountId = registerApsUniversalCreativeMount(container, descriptor())!; + const requestNonce = 'ZYXWVUTSRQPONMLKJIHGFE'; + + const request = () => + window.dispatchEvent( + new MessageEvent('message', { + data: { + message: 'trusted-server/aps/mount-request', + mountId, + nonce: requestNonce, + }, + source: requester.contentWindow, + }) + ); + request(); + request(); + + const rendererFrame = container.querySelector( + 'iframe[data-ts-aps-renderer="true"]' + )!; + expect(rendererFrame).not.toBeNull(); + expect(container.querySelectorAll('iframe[data-ts-aps-renderer="true"]')).toHaveLength(1); + expect(controller.isConnected).toBe(true); + + const { channel, sent } = advanceRendererToData(rendererFrame); + sendRendererMessage(channel, { + message: 'trusted-server/aps/renderer-ready', + nonce: sent.nonce, + }); + + expect(controller.isConnected).toBe(true); + expect(controller.style.display).toBe('none'); + expect(rendererFrame.style.display).toBe(''); + expect(resultPost).toHaveBeenCalledWith( + { + message: 'trusted-server/aps/mount-result', + mountId, + nonce: requestNonce, + status: 'ready', + }, + '*' + ); + document.body.innerHTML = ''; + }); + + it('revokes an older mount capability when the same container is registered again', () => { + document.body.innerHTML = '
'; + const container = document.getElementById('fictional-refresh-slot')!; + const requester = document.createElement('iframe'); + document.body.appendChild(requester); + const oldMountId = registerApsUniversalCreativeMount(container, descriptor())!; + const newMountId = registerApsUniversalCreativeMount(container, descriptor())!; + const request = (mountId: string) => + window.dispatchEvent( + new MessageEvent('message', { + data: { + message: 'trusted-server/aps/mount-request', + mountId, + nonce: 'ABCDEFGHIJKLMNOPQRSTUV', + }, + source: requester.contentWindow, + }) + ); + + request(oldMountId); + expect(container.querySelector('iframe[data-ts-aps-renderer="true"]')).toBeNull(); + request(newMountId); + const rendererFrame = container.querySelector( + 'iframe[data-ts-aps-renderer="true"]' + ); + expect(rendererFrame).not.toBeNull(); + rendererFrame!.dispatchEvent(new Event('error')); + document.body.innerHTML = ''; + }); + + it('resolves only after the top page acknowledges its one-shot mount request', async () => { const dynamicWindow = window as unknown as { - render?: (data: Record, helper: unknown, target: Window) => Promise; + render?: (data: Record) => Promise; }; + const postMessage = vi.spyOn(window.top, 'postMessage'); window.eval(APS_UNIVERSAL_CREATIVE_RENDERER); try { - const renderer = descriptor(); - const rendered = dynamicWindow.render!( - { - apsRenderer: renderer, - rendererUrl: apsRendererUrl(), - }, - undefined, - window - ); - const iframe = document.body.querySelector('iframe')!; - expect(iframe.src).toMatch(/\/integrations\/aps\/renderer#tsaps=[A-Za-z0-9_-]{22}$/); - expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); - expect(iframe.getAttribute('sandbox')).not.toContain('allow-same-origin'); - - const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); - iframe.dispatchEvent(new Event('load')); - const sent = postMessage.mock.calls[0][0] as { nonce: string; renderer: ApsRendererV1 }; - expect(sent.renderer).toEqual(renderer); + const mountId = 'ABCDEFGHIJKLMNOPQRSTUV'; + const rendered = dynamicWindow.render!({ + apsMountId: mountId, + publisherOrigin: window.location.origin, + }); + expect(document.body.querySelector('iframe')).toBeNull(); + expect(postMessage).toHaveBeenCalledTimes(1); + const sent = postMessage.mock.calls[0][0] as { + message: string; + mountId: string; + nonce: string; + }; + expect(sent).toEqual({ + message: 'trusted-server/aps/mount-request', + mountId, + nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), + }); let settled = false; void rendered.then(() => { @@ -488,12 +652,18 @@ describe('Universal Creative APS source', () => { window.dispatchEvent( new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, - source: iframe.contentWindow, + data: { + message: 'trusted-server/aps/mount-result', + mountId, + nonce: sent.nonce, + status: 'ready', + }, + source: window.top, }) ); await expect(rendered).resolves.toBeUndefined(); } finally { + postMessage.mockRestore(); delete dynamicWindow.render; document.body.innerHTML = ''; } diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 63caff94e..4a95d04e8 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -3176,11 +3176,12 @@ describe('installTsRenderBridge', () => { expect(Object.keys(response).sort()).toEqual( [ 'adId', + 'apsMountId', 'apsRenderer', 'height', 'message', + 'publisherOrigin', 'renderer', - 'rendererUrl', 'rendererVersion', 'width', ].sort() @@ -3189,8 +3190,9 @@ describe('installTsRenderBridge', () => { message: 'Prebid Response', adId: renderer.bidId, renderer: expect.stringContaining('window.render=function'), - rendererVersion: 4, - rendererUrl: new URL('/integrations/aps/renderer', window.location.origin).href, + rendererVersion: 6, + apsMountId: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), + publisherOrigin: window.location.origin, apsRenderer: renderer, width: 300, height: 250, @@ -3198,35 +3200,9 @@ describe('installTsRenderBridge', () => { expect(String(response.renderer)).not.toContain(renderer.accountId); expect(String(response.renderer)).not.toContain(renderer.aaxResponse); - // Universal Creative's dynamic-renderer path evaluates the returned static - // source and calls window.render(response, helper, targetWindow). Consume - // the exact bridge response through that deployed protocol shape. - const dynamicWindow = window as unknown as { - render?: (data: Record, helper: unknown, target: Window) => Promise; - }; - window.eval(String(response.renderer)); - try { - const rendered = dynamicWindow.render!(response, undefined, window); - const outerFrame = document.querySelector( - 'iframe[src*="/integrations/aps/renderer#tsaps="]' - )!; - expect(outerFrame).not.toBeNull(); - expect(outerFrame.getAttribute('sandbox')).not.toContain('allow-same-origin'); - - const rendererPost = vi.spyOn(outerFrame.contentWindow!, 'postMessage'); - outerFrame.dispatchEvent(new Event('load')); - const sent = rendererPost.mock.calls[0][0] as { nonce: string }; - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, - source: outerFrame.contentWindow, - }) - ); - await expect(rendered).resolves.toBeUndefined(); - outerFrame.remove(); - } finally { - delete dynamicWindow.render; - } + expect(String(response.renderer)).toContain('d&&d.apsMountId'); + expect(String(response.renderer)).not.toContain(renderer.creativeUrl); + expect(String(response.renderer)).not.toContain(renderer.aaxResponse); beaconSpy.mockRestore(); }); diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index 658f5d5bc..3ab6e904b 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -150,30 +150,27 @@ Trusted Server does not insert APS creative markup into the publisher document. Seats, `impid`, markup, notifications, user-sync data, sibling bids, losing seats, and unknown fields are not exposed. The browser decodes this envelope and cross-checks the ID, dimensions, URL, and tag type before any DOM mutation or message suppression. -Both rendering paths use `GET /integrations/aps/renderer`, a static Trusted Server document with its own restrictive CSP. The document initializes the account-keyed APS queue and then loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. +Both rendering paths use a publisher-origin bootstrap followed by two nested `data:` documents. TSJS first loads `GET /integrations/aps/renderer?mode=data-bootstrap` with a sandbox that omits `allow-same-origin`. The bootstrap is therefore opaque and cannot read or modify the publisher document. After a nonce-bound readiness message, TSJS adds `allow-same-origin` and asks the bootstrap to navigate itself to a per-impression `data:` container. The container then creates the static inner `data:` renderer under the same permanent sandbox. -The outer iframe uses these sandbox permissions: +Both data documents are naturally opaque even with `allow-same-origin`, so the publisher cannot access them. Keeping that token on both frames also avoids WebKit propagating an opaque origin to the HTTPS creative: the creative and its same-origin descendants retain their real origin in Chromium, Firefox, and WebKit. -```text -allow-forms -allow-pointer-lock -allow-popups -allow-popups-to-escape-sandbox -allow-scripts -allow-top-navigation-by-user-activation -``` +The outer container's CSP allows child frames from `data:` and only the fully validated creative URL's exact origin. That policy is inherited by the inner data renderer and intersects with the renderer's own CSP. It permits the expected creative frame but blocks the inner renderer from navigating itself to the publisher origin before a request is made. This exact-origin boundary intentionally blocks an immediate creative-frame redirect or same-frame navigation to a different origin; validate real APS inventory for intermediate or redirect origins before rollout. User-activated top navigation and popups remain governed by the sandbox tokens. + +A network-loaded HTTPS creative does not inherit its ancestor's CSP and can create further frames. To prevent it from using a publisher-origin descendant plus an executable publisher gadget to regain access to the top page, enabling APS appends a separate `Content-Security-Policy: frame-ancestors 'self'` policy to every Trusted Server response. Browsers require every ancestor to match, so publisher documents cannot load below the opaque container or third-party creative. This secure default also prevents the APS-enabled publisher from being embedded cross-origin; publishers that require trusted external embedders need a separately reviewed ancestor-allowlist feature before enabling APS. + +Trusted Server generates independent 128-bit nonces for the container and renderer. Once the inner renderer is ready, the container transfers a dedicated `MessagePort` between trusted top-page TSJS and the inner renderer. The complete descriptor travels only over that port; the inner renderer revalidates it, rejects the publisher origin, and requires the creative origin to equal the origin bound in the container CSP. The container URL contains only that exact origin, static renderer source, and nonces—never the response envelope, bid ID, price, or creative URL path. -It deliberately omits `allow-same-origin`, so APS and bidder execution remains below an opaque-origin boundary. The renderer response repeats these restrictions with a CSP `sandbox` directive, preventing another embedding path from restoring publisher-origin execution by omitting the iframe attribute. Trusted Server generates a fresh 128-bit nonce, binds it in the iframe URL fragment before navigation, and requires the same one-time nonce in the parent message and renderer acknowledgement. Existing slot content is retained until the static renderer has accepted the descriptor and loaded the fixed runner. +The inner document initializes the account-keyed APS queue and loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. A `renderer-ready` acknowledgement still means that Amazon's runner loaded, not that the final creative painted. Existing slot content remains until that acknowledgement. The query-free `GET /integrations/aps/renderer` response remains available with its original stricter CSP and legacy message shape for cached clients and rollback. ### Direct `/auction` -The TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. +The TSJS auction client validates the typed renderer descriptor and mounts the nested data renderer in the winning slot. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. ### GAM and Universal Creative -For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program that creates the same opaque renderer iframe. +For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program with a one-shot mount capability. That program asks trusted top-page TSJS to mount the nested data renderer as a sibling of the Universal Creative iframe, outside inherited GAM and Universal Creative sandbox restrictions. -For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes the capability once, and passes the APS bid ID separately to the Amazon runner. +For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes both the renderer and mount capabilities once, and passes the APS bid ID separately to the Amazon runner. These paths do not fetch PBS Cache, fire generic APS win/billing beacons, or call `apstag.setDisplayBids()` for the Trusted Server winner. Publisher-owned native APS objects are otherwise left untouched. @@ -182,10 +179,10 @@ These paths do not fetch PBS Cache, fire generic APS win/billing beacons, or cal The publisher policy must permit the same-origin renderer route, for example: ```text -frame-src 'self' +frame-src 'self' data: ``` -Do not add `allow-same-origin` to the outer renderer sandbox. The renderer endpoint supplies its own CSP for the fixed runner and HTTPS creative resources. The same-origin renderer route inherits the publisher page scheme; use HTTPS in production. APS endpoints and third-party creative URLs always require HTTPS. +The `data:` source is required for the bootstrap's self-navigation to the opaque container. APS also adds `frame-ancestors 'self'` as an independent response policy; do not override or remove that policy. Do not weaken or bypass the initial bootstrap sandbox. TSJS adds `allow-same-origin` only when navigating away from the publisher-origin bootstrap; both final data documents remain naturally opaque. The bootstrap response supplies the resource CSP for the fixed runner and HTTPS creative resources, while the container narrows `frame-src` to the validated creative origin. The same-origin bootstrap route inherits the publisher page scheme; use HTTPS in production. APS endpoints and third-party creative URLs always require HTTPS. Before enabling script creatives, verify under the publisher's actual CSP that both iframe and script-tag creatives: @@ -235,8 +232,8 @@ Use fictional values in source-controlled configuration and fixtures. Supply con ### Winner targets but does not render -- Confirm `GET /integrations/aps/renderer` returns HTML with its CSP and `Referrer-Policy: no-referrer`. -- Confirm publisher CSP permits `frame-src 'self'`. +- Confirm `GET /integrations/aps/renderer?mode=data-bootstrap` returns HTML with its bootstrap CSP and `Referrer-Policy: no-referrer`. +- Confirm publisher CSP permits `frame-src 'self' data:`. - Confirm the GAM creative uses the supported Prebid Universal Creative bridge and the winning `hb_adid`. - For client-side `trustedServer` adapter auctions, confirm Prebid's `bidResponse` contains a generated `adId` and that the corresponding capability appears briefly in `window.tsjs.apsPrebidRenderers` before rendering. - Ensure no native APS path is trying to handle the same cohort. From 104d163569fe9953a3c73e1231c32b7de13bee72 Mon Sep 17 00:00:00 2001 From: Christian Date: Tue, 18 Aug 2026 11:50:33 -0500 Subject: [PATCH 331/395] Revert "Reapply "Merge remote-tracking branch 'origin/pr/1033' into rc/202608"" This reverts commit 3953cd6b31217d986973c9c525ebfa5da3c0fe33. --- .../src/integrations/aps.rs | 125 ++-- .../src/response_privacy.rs | 64 -- ...prebid-universal-creative-1.17.2-banner.js | 4 - .../browser/tests/shared/aps-renderer.spec.ts | 706 +----------------- .../trusted-server-js/lib/src/core/request.ts | 3 +- .../lib/src/integrations/aps/render.ts | 401 ++-------- .../integrations/aps/renderer-bootstrap.html | 41 - .../integrations/aps/renderer-container.html | 71 -- .../lib/src/integrations/aps/renderer.html | 228 ------ .../lib/src/integrations/gpt/index.ts | 36 +- .../lib/test/core/request.test.ts | 48 +- .../lib/test/integrations/aps/render.test.ts | 334 ++------- .../lib/test/integrations/gpt/ad_init.test.ts | 40 +- docs/guide/integrations/aps.md | 33 +- 14 files changed, 314 insertions(+), 1820 deletions(-) delete mode 100644 crates/trusted-server-integration-tests/browser/fixtures/prebid-universal-creative-1.17.2-banner.js delete mode 100644 crates/trusted-server-js/lib/src/integrations/aps/renderer-bootstrap.html delete mode 100644 crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html delete mode 100644 crates/trusted-server-js/lib/src/integrations/aps/renderer.html diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 6121d6261..4fed9278d 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -36,7 +36,6 @@ use crate::settings::{IntegrationConfig, Settings}; const APS_INTEGRATION_ID: &str = "aps"; const APS_RENDERER_ROUTE: &str = "/integrations/aps/renderer"; -const APS_RENDERER_BOOTSTRAP_QUERY: &str = "mode=data-bootstrap"; const DEFAULT_CURRENCY: &str = "USD"; const APS_SDK_SOURCE: &str = "prebid"; const APS_SDK_VERSION: &str = "2.2.0"; @@ -48,12 +47,72 @@ const MAX_LANGUAGE_BYTES: usize = 8; const MAX_PAGE_URL_BYTES: usize = 8192; const MAX_RENDER_ENVELOPE_BYTES: usize = 256 * 1024; const APS_RENDERER_CSP: &str = "default-src 'none'; sandbox allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation; script-src 'unsafe-inline' https:; connect-src https:; frame-src https:; img-src https: data:; media-src https: blob:; style-src 'unsafe-inline' https:; font-src https: data:;"; -const APS_RENDERER_BOOTSTRAP_CSP: &str = "default-src 'none'; sandbox allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-top-navigation-by-user-activation; script-src 'unsafe-inline' https:; connect-src https:; frame-src https: data:; img-src https: data:; media-src https: blob:; style-src 'unsafe-inline' https:; font-src https: data:;"; -const APS_RENDERER_DOCUMENT: &str = - include_str!("../../../trusted-server-js/lib/src/integrations/aps/renderer.html"); -const APS_RENDERER_BOOTSTRAP_DOCUMENT: &str = - include_str!("../../../trusted-server-js/lib/src/integrations/aps/renderer-bootstrap.html"); +const APS_RENDERER_DOCUMENT: &str = r#" + + +"#; /// Configuration for the APS `OpenRTB` integration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] @@ -1152,28 +1211,13 @@ impl IntegrationProxy for ApsRendererIntegration { message: "Failed to build APS not-found response".to_string(), }); } - let (renderer_document, renderer_csp) = match request.uri().query() { - None => (APS_RENDERER_DOCUMENT, APS_RENDERER_CSP), - Some(APS_RENDERER_BOOTSTRAP_QUERY) => { - (APS_RENDERER_BOOTSTRAP_DOCUMENT, APS_RENDERER_BOOTSTRAP_CSP) - } - Some(_) => { - return http::Response::builder() - .status(StatusCode::NOT_FOUND) - .body(EdgeBody::from("Not Found")) - .change_context(TrustedServerError::Integration { - integration: APS_INTEGRATION_ID.to_string(), - message: "Failed to build APS not-found response".to_string(), - }); - } - }; http::Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "text/html; charset=utf-8") .header("x-content-type-options", "nosniff") .header("referrer-policy", "no-referrer") - .header(header::CONTENT_SECURITY_POLICY, renderer_csp) - .body(EdgeBody::from(renderer_document)) + .header(header::CONTENT_SECURITY_POLICY, APS_RENDERER_CSP) + .body(EdgeBody::from(APS_RENDERER_DOCUMENT)) .change_context(TrustedServerError::Integration { integration: APS_INTEGRATION_ID.to_string(), message: "Failed to build APS renderer response".to_string(), @@ -2274,7 +2318,7 @@ mod tests { } #[test] - fn registers_and_serves_static_renderer_and_data_bootstrap_modes() { + fn registers_and_serves_only_static_renderer_route() { let integration = ApsRendererIntegration; let routes = integration.routes(); assert_eq!(routes.len(), 1, "should register one route"); @@ -2303,25 +2347,6 @@ mod tests { APS_RENDERER_CSP ); - let bootstrap = http::Request::builder() - .method(Method::GET) - .uri(format!( - "{APS_RENDERER_ROUTE}?{APS_RENDERER_BOOTSTRAP_QUERY}" - )) - .body(EdgeBody::empty()) - .expect("should build renderer bootstrap request"); - let response = - futures::executor::block_on(integration.handle(&settings, &services, bootstrap)) - .expect("should serve renderer bootstrap"); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - response.headers()[header::CONTENT_SECURITY_POLICY], - APS_RENDERER_BOOTSTRAP_CSP - ); - assert!(APS_RENDERER_BOOTSTRAP_CSP.contains("allow-same-origin")); - assert!(APS_RENDERER_BOOTSTRAP_CSP.contains("frame-src https: data:")); - assert!(APS_RENDERER_BOOTSTRAP_DOCUMENT.contains("trusted-server/aps/bootstrap-navigate")); - let post = http::Request::builder() .method(Method::POST) .uri(APS_RENDERER_ROUTE) @@ -2349,7 +2374,6 @@ mod tests { assert_eq!(registration.integration_id, APS_INTEGRATION_ID); assert_eq!(registration.proxies.len(), 1); - assert!(registration.request_filters.is_empty()); assert!(registration.js_disabled); } @@ -2401,15 +2425,12 @@ mod tests { #[test] fn renderer_document_is_static_and_nonce_bound() { assert!(APS_RENDERER_DOCUMENT.contains("^#tsaps=")); - assert!(APS_RENDERER_DOCUMENT.contains("event.source !== parent")); - assert!(APS_RENDERER_DOCUMENT.contains("message.nonce !== expected")); - assert!(APS_RENDERER_DOCUMENT.contains("['nonce', 'publisherOrigin', 'renderer']")); - assert!(APS_RENDERER_DOCUMENT.contains("['nonce', 'renderer']")); - assert!(APS_RENDERER_DOCUMENT.contains("url.origin === publisherOrigin")); + assert!(APS_RENDERER_DOCUMENT.contains("event.source!==parent")); + assert!(APS_RENDERER_DOCUMENT.contains("message.nonce!==expected")); assert!(APS_RENDERER_DOCUMENT.contains("prebid/creative/render")); assert!(APS_RENDERER_DOCUMENT.contains("window._aps instanceof Map")); - assert!(APS_RENDERER_DOCUMENT.contains("store: new Map([['listeners', new Map()]])")); - assert!(APS_RENDERER_DOCUMENT.contains("account.queue.push(")); + assert!(APS_RENDERER_DOCUMENT.contains("store:new Map([['listeners',new Map()]])")); + assert!(APS_RENDERER_DOCUMENT.contains("account.queue.push(new CustomEvent")); assert!( APS_RENDERER_DOCUMENT.contains("trusted-server/aps/renderer-ready") && APS_RENDERER_DOCUMENT.contains("trusted-server/aps/renderer-failed") @@ -2421,7 +2442,7 @@ mod tests { ); assert!(!APS_RENDERER_DOCUMENT.contains(" diff --git a/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html b/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html deleted file mode 100644 index 6ef871c9b..000000000 --- a/crates/trusted-server-js/lib/src/integrations/aps/renderer-container.html +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - - - diff --git a/crates/trusted-server-js/lib/src/integrations/aps/renderer.html b/crates/trusted-server-js/lib/src/integrations/aps/renderer.html deleted file mode 100644 index b7ddac1c0..000000000 --- a/crates/trusted-server-js/lib/src/integrations/aps/renderer.html +++ /dev/null @@ -1,228 +0,0 @@ - - - - - diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index ef6a3204c..52a6ecb70 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -10,9 +10,9 @@ import type { import { APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, + apsRendererUrl, consumeApsPrebidRenderer, getApsPrebidRenderer, - registerApsUniversalCreativeMount, validateApsRenderer, } from '../aps/render'; @@ -225,14 +225,13 @@ function slotIdForMessageSource(source: MessageEventSource | null): string | und ?.id; } -function slotRootForMessageSource( +function messageSourceBelongsToAdUnit( source: MessageEventSource | null, - divId: string -): HTMLElement | undefined { - if (!source) return undefined; - return candidateSlotRootsForConfiguredDivId(divId).find((root) => - sourceIsInSlotRoots(source, [root]) - ); + adUnitCode: string +): boolean { + return source + ? sourceIsInSlotRoots(source, candidateSlotRootsForConfiguredDivId(adUnitCode)) + : false; } function clearTargetingKeys(slot: GoogleTagSlot, keys: Iterable): void { @@ -1701,15 +1700,13 @@ export function installTsRenderBridge(): void { // Prebid handles ad IDs globally and would otherwise answer a request from // an unrelated iframe when this slot-bound capability rejects it. e.stopImmediatePropagation(); - const mountContainer = slotRootForMessageSource(e.source, prebidRendererEntry.adUnitCode); - if (!mountContainer) return; + if (!messageSourceBelongsToAdUnit(e.source, prebidRendererEntry.adUnitCode)) return; const renderer = validateApsRenderer(prebidRendererEntry.renderer); - if (!renderer) return; + const rendererUrl = apsRendererUrl(); + if (!renderer || !rendererUrl) return; if (!hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; if (!consumeApsPrebidRenderer(adId, prebidRendererEntry)) return; recordConsumedPrebidApsId(consumedPrebidApsIds, adId, prebidRendererEntry.expiresAt); - const apsMountId = registerApsUniversalCreativeMount(mountContainer, renderer); - if (!apsMountId) return; port.postMessage( JSON.stringify({ @@ -1717,8 +1714,7 @@ export function installTsRenderBridge(): void { adId, renderer: APS_UNIVERSAL_CREATIVE_RENDERER, rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - apsMountId, - publisherOrigin: window.location.origin, + rendererUrl, apsRenderer: renderer, width: renderer.width, height: renderer.height, @@ -1757,11 +1753,8 @@ export function installTsRenderBridge(): void { e.stopImmediatePropagation(); if (consumedServerApsBySlot.get(slotId) === adId) return; const renderer = validateApsRenderer(matchedBid.renderer); - const configuredSlot = window.tsjs?.adSlots?.find((slot) => slot.id === slotId); - const mountContainer = slotRootForMessageSource(e.source, configuredSlot?.div_id ?? slotId); - if (!renderer || !mountContainer) return; - const apsMountId = registerApsUniversalCreativeMount(mountContainer, renderer); - if (!apsMountId) return; + const rendererUrl = apsRendererUrl(); + if (!renderer || !rendererUrl) return; consumedServerApsBySlot.set(slotId, adId); port.postMessage( JSON.stringify({ @@ -1769,8 +1762,7 @@ export function installTsRenderBridge(): void { adId, renderer: APS_UNIVERSAL_CREATIVE_RENDERER, rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - apsMountId, - publisherOrigin: window.location.origin, + rendererUrl, apsRenderer: renderer, width: renderer.width, height: renderer.height, diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index ba2dfb274..dc17c9e87 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -69,7 +69,7 @@ describe('request.requestAds', () => { ); }); - it('dispatches a valid APS descriptor through the opaque data renderer bootstrap', async () => { + it('dispatches a valid APS descriptor to the opaque static renderer route', async () => { const apsBid = envelope.seatbid[0].bid[0]; const renderer = { type: 'aps', @@ -117,55 +117,21 @@ describe('request.requestAds', () => { const iframe = document.querySelector('#slot1 iframe') as HTMLIFrameElement | null; expect(iframe).not.toBeNull(); - expect(iframe!.src).toContain('/integrations/aps/renderer?mode=data-bootstrap#tsaps='); + expect(iframe!.src).toContain('/integrations/aps/renderer#tsaps='); expect(iframe!.srcdoc).toBe(''); expect(iframe!.getAttribute('sandbox')).not.toContain('allow-same-origin'); expect(document.querySelector('#slot1 span')).not.toBeNull(); const postMessage = vi.spyOn(iframe!.contentWindow!, 'postMessage'); - const nonce = new URL(iframe!.src).hash.replace('#tsaps=', ''); - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/bootstrap-ready', nonce }, - source: iframe!.contentWindow, - }) - ); - const navigate = postMessage.mock.calls[0][0] as { rendererUrl: string }; - const containerDocument = decodeURIComponent( - navigate.rendererUrl - .slice('data:text/html;charset=utf-8,'.length) - .replace(/#tsaps=[A-Za-z0-9_-]{22}$/, '') - ); - const innerNonce = containerDocument.match( - /data:text\/html;charset=utf-8,.+#tsaps=([A-Za-z0-9_-]{22})/ - )?.[1]; - expect(innerNonce).toMatch(/^[A-Za-z0-9_-]{22}$/); - - const channel = { - close: vi.fn(), - onmessage: null, - postMessage: vi.fn(), - start: vi.fn(), - } as unknown as MessagePort; - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/container-ready', nonce }, - source: iframe!.contentWindow, - ports: [channel], - }) - ); - channel.onmessage?.( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/channel-ready', nonce: innerNonce }, - }) - ); + iframe!.dispatchEvent(new Event('load')); expect(document.querySelector('#slot1 span')).not.toBeNull(); - expect(channel.postMessage).toHaveBeenCalledWith(expect.objectContaining({ renderer })); + expect(postMessage).toHaveBeenCalledWith(expect.objectContaining({ renderer }), '*'); - const message = vi.mocked(channel.postMessage).mock.calls[0][0] as { nonce: string }; - channel.onmessage?.( + const message = postMessage.mock.calls[0][0] as { nonce: string }; + window.dispatchEvent( new MessageEvent('message', { data: { message: 'trusted-server/aps/renderer-ready', nonce: message.nonce }, + source: iframe!.contentWindow, }) ); expect(document.querySelector('#slot1 span')).toBeNull(); diff --git a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts index c9808b8ac..eae60c90e 100644 --- a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts @@ -4,18 +4,14 @@ import envelope from '../../fixtures/aps-renderer-v1.json'; import type { ApsRendererV1 } from '../../../src/core/types'; import { log } from '../../../src/core/log'; import { - APS_RENDERER_DATA_URL, APS_RENDERER_PATH, APS_RENDERER_SANDBOX, APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - apsRendererBootstrapUrl, apsRendererUrl, - cancelPendingApsRender, getApsPrebidRenderer, parseApsRendererDescriptor, registerApsPrebidRenderer, - registerApsUniversalCreativeMount, renderApsCreative, validateApsRenderer, } from '../../../src/integrations/aps/render'; @@ -54,76 +50,6 @@ function descriptor(overrides: Partial = {}): ApsRendererV1 { }; } -type FakeRendererChannel = MessagePort & { - close: ReturnType; - postMessage: ReturnType; - start: ReturnType; -}; - -function sendRendererMessage(channel: FakeRendererChannel, data: Record): void { - channel.onmessage?.(new MessageEvent('message', { data })); -} - -function advanceRendererToData(iframe: HTMLIFrameElement) { - const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); - const nonce = new URL(iframe.src).hash.replace('#tsaps=', ''); - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/bootstrap-ready', nonce }, - source: iframe.contentWindow, - }) - ); - expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); - const navigate = postMessage.mock.calls[0][0] as { - message: string; - nonce: string; - rendererUrl: string; - }; - expect(navigate.message).toBe('trusted-server/aps/bootstrap-navigate'); - expect(navigate.nonce).toBe(nonce); - expect(navigate.rendererUrl).toMatch( - /^data:text\/html;charset=utf-8,.+#tsaps=[A-Za-z0-9_-]{22}$/ - ); - - const encodedDocument = navigate.rendererUrl - .slice('data:text/html;charset=utf-8,'.length) - .replace(/#tsaps=[A-Za-z0-9_-]{22}$/, ''); - const containerDocument = decodeURIComponent(encodedDocument); - const innerNonce = containerDocument.match( - /data:text\/html;charset=utf-8,.+#tsaps=([A-Za-z0-9_-]{22})/ - )?.[1]; - expect(innerNonce).toMatch(/^[A-Za-z0-9_-]{22}$/); - expect(innerNonce).not.toBe(nonce); - expect(containerDocument).toContain('frame-src data: https://creative.example'); - expect(containerDocument).not.toContain(descriptor().bidId); - expect(containerDocument).not.toContain(descriptor().aaxResponse); - - const channel = { - close: vi.fn(), - onmessage: null, - postMessage: vi.fn(), - start: vi.fn(), - } as unknown as FakeRendererChannel; - window.dispatchEvent( - new MessageEvent('message', { - data: { message: 'trusted-server/aps/container-ready', nonce }, - source: iframe.contentWindow, - ports: [channel], - }) - ); - expect(channel.start).toHaveBeenCalledOnce(); - sendRendererMessage(channel, { - message: 'trusted-server/aps/channel-ready', - nonce: innerNonce, - }); - const sent = channel.postMessage.mock.calls[0][0] as { - nonce: string; - publisherOrigin: string; - renderer: ApsRendererV1; - }; - return { channel, innerNonce: innerNonce!, postMessage, sent }; -} - describe('APS renderer validation', () => { it('consumes the shared fictional golden envelope and supports an omitted creative ID', () => { const withCreativeId = descriptor(); @@ -345,58 +271,62 @@ describe('direct APS rendering', () => { document.body.innerHTML = ''; }); - it('bootstraps the data renderer with a fragment-bound 128-bit nonce', () => { + it('loads the static route with a fragment-bound 128-bit nonce and opaque sandbox', () => { expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const slot = document.getElementById('fictional-slot')!; const iframe = slot.querySelector('iframe')!; const existing = slot.querySelector('span'); expect(existing).not.toBeNull(); - expect(iframe.src.startsWith(`${apsRendererBootstrapUrl()}#tsaps=`)).toBe(true); - expect(iframe.src).toMatch(/\?mode=data-bootstrap#tsaps=[A-Za-z0-9_-]{22}$/); + expect(iframe.src).toMatch(/\/integrations\/aps\/renderer#tsaps=[A-Za-z0-9_-]{22}$/); + expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); expect(iframe.getAttribute('sandbox')).not.toContain('allow-same-origin'); expect(iframe.srcdoc).toBe(''); - const { channel, sent } = advanceRendererToData(iframe); + const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); + iframe.dispatchEvent(new Event('load')); + expect(slot.querySelector('span')).not.toBeNull(); expect(iframe.style.display).toBe('none'); - expect(iframe.getAttribute('sandbox')).toContain('allow-same-origin'); - expect(sent).toEqual({ - nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), - publisherOrigin: window.location.origin, - renderer: descriptor(), - }); - - sendRendererMessage(channel, { - message: 'trusted-server/aps/renderer-ready', - nonce: `wrong-${sent.nonce}`, - }); - expect(slot.querySelector('span')).not.toBeNull(); + expect(postMessage).toHaveBeenCalledTimes(1); + expect(postMessage).toHaveBeenCalledWith( + { + nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), + renderer: descriptor(), + }, + '*' + ); + const message = postMessage.mock.calls[0][0] as { nonce: string }; window.dispatchEvent( new MessageEvent('message', { - data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, + data: { + message: 'trusted-server/aps/renderer-ready', + nonce: `wrong-${message.nonce}`, + }, source: iframe.contentWindow, }) ); expect(slot.querySelector('span')).not.toBeNull(); - expect(iframe.style.display).toBe('none'); - sendRendererMessage(channel, { - message: 'trusted-server/aps/renderer-ready', - nonce: sent.nonce, - }); - expect(iframe.getAttribute('sandbox')).toContain('allow-same-origin'); + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/renderer-ready', nonce: message.nonce }, + source: iframe.contentWindow, + }) + ); expect(slot.querySelector('span')).toBeNull(); expect(iframe.style.display).toBe(''); }); - it('accepts readiness only through the transferred renderer channel', () => { + it('rejects a ready message with the correct nonce from a foreign window', () => { expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const slot = document.getElementById('fictional-slot')!; const rendererFrame = slot.querySelector('iframe')!; - const { channel, sent } = advanceRendererToData(rendererFrame); + const postMessage = vi.spyOn(rendererFrame.contentWindow!, 'postMessage'); + rendererFrame.dispatchEvent(new Event('load')); + const sent = postMessage.mock.calls[0][0] as { nonce: string }; const foreignFrame = document.createElement('iframe'); document.body.appendChild(foreignFrame); @@ -406,6 +336,10 @@ describe('direct APS rendering', () => { source: foreignFrame.contentWindow, }) ); + + expect(slot.querySelector('span')).not.toBeNull(); + expect(rendererFrame.style.display).toBe('none'); + window.dispatchEvent( new MessageEvent('message', { data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, @@ -413,13 +347,6 @@ describe('direct APS rendering', () => { }) ); - expect(slot.querySelector('span')).not.toBeNull(); - expect(rendererFrame.style.display).toBe('none'); - - sendRendererMessage(channel, { - message: 'trusted-server/aps/renderer-ready', - nonce: sent.nonce, - }); expect(slot.querySelector('span')).toBeNull(); expect(rendererFrame.style.display).toBe(''); }); @@ -441,16 +368,6 @@ describe('direct APS rendering', () => { expect(document.querySelector('#fictional-slot iframe')).toBeNull(); }); - it('cancels a pending frame before another renderer replaces the slot', () => { - expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); - const container = document.getElementById('fictional-slot')!; - - cancelPendingApsRender(container); - - expect(container.querySelector('span')).not.toBeNull(); - expect(container.querySelector('iframe')).toBeNull(); - }); - it('removes an unacknowledged frame without clearing publisher content', () => { vi.useFakeTimers(); try { @@ -474,9 +391,9 @@ describe('direct APS rendering', () => { const baselineTimers = vi.getTimerCount(); expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); const firstFrame = document.querySelector('#fictional-slot iframe')!; - const { channel: firstChannel, sent: firstSent } = advanceRendererToData( - firstFrame as HTMLIFrameElement - ); + const firstPostMessage = vi.spyOn(firstFrame.contentWindow!, 'postMessage'); + firstFrame.dispatchEvent(new Event('load')); + const firstSent = firstPostMessage.mock.calls[0][0] as { nonce: string }; const timersAfterFirst = vi.getTimerCount(); expect(timersAfterFirst).toBeGreaterThan(baselineTimers); @@ -484,21 +401,25 @@ describe('direct APS rendering', () => { const secondFrame = document.querySelector('#fictional-slot iframe')!; expect(firstFrame.isConnected).toBe(false); expect(vi.getTimerCount()).toBe(timersAfterFirst); - const { channel: secondChannel, sent } = advanceRendererToData( - secondFrame as HTMLIFrameElement - ); + const postMessage = vi.spyOn(secondFrame.contentWindow!, 'postMessage'); + secondFrame.dispatchEvent(new Event('load')); + const sent = postMessage.mock.calls[0][0] as { nonce: string }; - sendRendererMessage(firstChannel, { - message: 'trusted-server/aps/renderer-ready', - nonce: firstSent.nonce, - }); + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/renderer-ready', nonce: firstSent.nonce }, + source: firstFrame.contentWindow, + }) + ); expect(document.querySelector('#fictional-slot span')).not.toBeNull(); expect((secondFrame as HTMLIFrameElement).style.display).toBe('none'); - sendRendererMessage(secondChannel, { - message: 'trusted-server/aps/renderer-ready', - nonce: sent.nonce, - }); + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, + source: secondFrame.contentWindow, + }) + ); vi.advanceTimersByTime(10_000); expect(warnSpy).not.toHaveBeenCalled(); @@ -509,21 +430,19 @@ describe('direct APS rendering', () => { }); describe('Universal Creative APS source', () => { - it('uses the deployed dynamic renderer protocol to request a top-page mount', () => { - expect(APS_UNIVERSAL_CREATIVE_RENDERER_VERSION).toBeGreaterThanOrEqual(6); + it('uses the deployed dynamic renderer protocol and only creates the opaque route frame', () => { + expect(APS_UNIVERSAL_CREATIVE_RENDERER_VERSION).toBeGreaterThanOrEqual(4); expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('window.render=function'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.apsMountId'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.publisherOrigin'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('trusted-server/aps/mount-request'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('d&&d.apsRenderer'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('d&&d.rendererUrl'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(APS_RENDERER_DATA_URL); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(APS_RENDERER_SANDBOX); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.apsRenderer'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain('d&&d.rendererUrl'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain(APS_RENDERER_PATH); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).toContain(APS_RENDERER_SANDBOX); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('allow-same-origin'); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('srcdoc'); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('document.write'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('creativeUrl'); + expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('aaxResponse'); expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain('example-account-id'); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(descriptor().creativeUrl); - expect(APS_UNIVERSAL_CREATIVE_RENDERER).not.toContain(descriptor().aaxResponse); }); it('computes an absolute renderer URL from the publisher origin', () => { @@ -534,114 +453,31 @@ describe('Universal Creative APS source', () => { expect(apsRendererUrl('not an origin')).toBeUndefined(); }); - it('consumes a top-page mount capability once and preserves the controller frame', () => { - document.body.innerHTML = - '
'; - const container = document.getElementById('fictional-puc-slot')!; - const controller = container.querySelector('.puc-controller')!; - const requester = document.createElement('iframe'); - document.body.appendChild(requester); - const resultPost = vi.spyOn(requester.contentWindow!, 'postMessage'); - const mountId = registerApsUniversalCreativeMount(container, descriptor())!; - const requestNonce = 'ZYXWVUTSRQPONMLKJIHGFE'; - - const request = () => - window.dispatchEvent( - new MessageEvent('message', { - data: { - message: 'trusted-server/aps/mount-request', - mountId, - nonce: requestNonce, - }, - source: requester.contentWindow, - }) - ); - request(); - request(); - - const rendererFrame = container.querySelector( - 'iframe[data-ts-aps-renderer="true"]' - )!; - expect(rendererFrame).not.toBeNull(); - expect(container.querySelectorAll('iframe[data-ts-aps-renderer="true"]')).toHaveLength(1); - expect(controller.isConnected).toBe(true); - - const { channel, sent } = advanceRendererToData(rendererFrame); - sendRendererMessage(channel, { - message: 'trusted-server/aps/renderer-ready', - nonce: sent.nonce, - }); - - expect(controller.isConnected).toBe(true); - expect(controller.style.display).toBe('none'); - expect(rendererFrame.style.display).toBe(''); - expect(resultPost).toHaveBeenCalledWith( - { - message: 'trusted-server/aps/mount-result', - mountId, - nonce: requestNonce, - status: 'ready', - }, - '*' - ); - document.body.innerHTML = ''; - }); - - it('revokes an older mount capability when the same container is registered again', () => { - document.body.innerHTML = '
'; - const container = document.getElementById('fictional-refresh-slot')!; - const requester = document.createElement('iframe'); - document.body.appendChild(requester); - const oldMountId = registerApsUniversalCreativeMount(container, descriptor())!; - const newMountId = registerApsUniversalCreativeMount(container, descriptor())!; - const request = (mountId: string) => - window.dispatchEvent( - new MessageEvent('message', { - data: { - message: 'trusted-server/aps/mount-request', - mountId, - nonce: 'ABCDEFGHIJKLMNOPQRSTUV', - }, - source: requester.contentWindow, - }) - ); - - request(oldMountId); - expect(container.querySelector('iframe[data-ts-aps-renderer="true"]')).toBeNull(); - request(newMountId); - const rendererFrame = container.querySelector( - 'iframe[data-ts-aps-renderer="true"]' - ); - expect(rendererFrame).not.toBeNull(); - rendererFrame!.dispatchEvent(new Event('error')); - document.body.innerHTML = ''; - }); - - it('resolves only after the top page acknowledges its one-shot mount request', async () => { + it('creates the opaque route frame and resolves only after the bound acknowledgement', async () => { const dynamicWindow = window as unknown as { - render?: (data: Record) => Promise; + render?: (data: Record, helper: unknown, target: Window) => Promise; }; - const postMessage = vi.spyOn(window.top, 'postMessage'); window.eval(APS_UNIVERSAL_CREATIVE_RENDERER); try { - const mountId = 'ABCDEFGHIJKLMNOPQRSTUV'; - const rendered = dynamicWindow.render!({ - apsMountId: mountId, - publisherOrigin: window.location.origin, - }); - expect(document.body.querySelector('iframe')).toBeNull(); - expect(postMessage).toHaveBeenCalledTimes(1); - const sent = postMessage.mock.calls[0][0] as { - message: string; - mountId: string; - nonce: string; - }; - expect(sent).toEqual({ - message: 'trusted-server/aps/mount-request', - mountId, - nonce: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), - }); + const renderer = descriptor(); + const rendered = dynamicWindow.render!( + { + apsRenderer: renderer, + rendererUrl: apsRendererUrl(), + }, + undefined, + window + ); + const iframe = document.body.querySelector('iframe')!; + expect(iframe.src).toMatch(/\/integrations\/aps\/renderer#tsaps=[A-Za-z0-9_-]{22}$/); + expect(iframe.getAttribute('sandbox')).toBe(APS_RENDERER_SANDBOX); + expect(iframe.getAttribute('sandbox')).not.toContain('allow-same-origin'); + + const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); + iframe.dispatchEvent(new Event('load')); + const sent = postMessage.mock.calls[0][0] as { nonce: string; renderer: ApsRendererV1 }; + expect(sent.renderer).toEqual(renderer); let settled = false; void rendered.then(() => { @@ -652,18 +488,12 @@ describe('Universal Creative APS source', () => { window.dispatchEvent( new MessageEvent('message', { - data: { - message: 'trusted-server/aps/mount-result', - mountId, - nonce: sent.nonce, - status: 'ready', - }, - source: window.top, + data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, + source: iframe.contentWindow, }) ); await expect(rendered).resolves.toBeUndefined(); } finally { - postMessage.mockRestore(); delete dynamicWindow.render; document.body.innerHTML = ''; } diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 4a95d04e8..63caff94e 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -3176,12 +3176,11 @@ describe('installTsRenderBridge', () => { expect(Object.keys(response).sort()).toEqual( [ 'adId', - 'apsMountId', 'apsRenderer', 'height', 'message', - 'publisherOrigin', 'renderer', + 'rendererUrl', 'rendererVersion', 'width', ].sort() @@ -3190,9 +3189,8 @@ describe('installTsRenderBridge', () => { message: 'Prebid Response', adId: renderer.bidId, renderer: expect.stringContaining('window.render=function'), - rendererVersion: 6, - apsMountId: expect.stringMatching(/^[A-Za-z0-9_-]{22}$/), - publisherOrigin: window.location.origin, + rendererVersion: 4, + rendererUrl: new URL('/integrations/aps/renderer', window.location.origin).href, apsRenderer: renderer, width: 300, height: 250, @@ -3200,9 +3198,35 @@ describe('installTsRenderBridge', () => { expect(String(response.renderer)).not.toContain(renderer.accountId); expect(String(response.renderer)).not.toContain(renderer.aaxResponse); - expect(String(response.renderer)).toContain('d&&d.apsMountId'); - expect(String(response.renderer)).not.toContain(renderer.creativeUrl); - expect(String(response.renderer)).not.toContain(renderer.aaxResponse); + // Universal Creative's dynamic-renderer path evaluates the returned static + // source and calls window.render(response, helper, targetWindow). Consume + // the exact bridge response through that deployed protocol shape. + const dynamicWindow = window as unknown as { + render?: (data: Record, helper: unknown, target: Window) => Promise; + }; + window.eval(String(response.renderer)); + try { + const rendered = dynamicWindow.render!(response, undefined, window); + const outerFrame = document.querySelector( + 'iframe[src*="/integrations/aps/renderer#tsaps="]' + )!; + expect(outerFrame).not.toBeNull(); + expect(outerFrame.getAttribute('sandbox')).not.toContain('allow-same-origin'); + + const rendererPost = vi.spyOn(outerFrame.contentWindow!, 'postMessage'); + outerFrame.dispatchEvent(new Event('load')); + const sent = rendererPost.mock.calls[0][0] as { nonce: string }; + window.dispatchEvent( + new MessageEvent('message', { + data: { message: 'trusted-server/aps/renderer-ready', nonce: sent.nonce }, + source: outerFrame.contentWindow, + }) + ); + await expect(rendered).resolves.toBeUndefined(); + outerFrame.remove(); + } finally { + delete dynamicWindow.render; + } beaconSpy.mockRestore(); }); diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index 3ab6e904b..658f5d5bc 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -150,27 +150,30 @@ Trusted Server does not insert APS creative markup into the publisher document. Seats, `impid`, markup, notifications, user-sync data, sibling bids, losing seats, and unknown fields are not exposed. The browser decodes this envelope and cross-checks the ID, dimensions, URL, and tag type before any DOM mutation or message suppression. -Both rendering paths use a publisher-origin bootstrap followed by two nested `data:` documents. TSJS first loads `GET /integrations/aps/renderer?mode=data-bootstrap` with a sandbox that omits `allow-same-origin`. The bootstrap is therefore opaque and cannot read or modify the publisher document. After a nonce-bound readiness message, TSJS adds `allow-same-origin` and asks the bootstrap to navigate itself to a per-impression `data:` container. The container then creates the static inner `data:` renderer under the same permanent sandbox. +Both rendering paths use `GET /integrations/aps/renderer`, a static Trusted Server document with its own restrictive CSP. The document initializes the account-keyed APS queue and then loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. -Both data documents are naturally opaque even with `allow-same-origin`, so the publisher cannot access them. Keeping that token on both frames also avoids WebKit propagating an opaque origin to the HTTPS creative: the creative and its same-origin descendants retain their real origin in Chromium, Firefox, and WebKit. +The outer iframe uses these sandbox permissions: -The outer container's CSP allows child frames from `data:` and only the fully validated creative URL's exact origin. That policy is inherited by the inner data renderer and intersects with the renderer's own CSP. It permits the expected creative frame but blocks the inner renderer from navigating itself to the publisher origin before a request is made. This exact-origin boundary intentionally blocks an immediate creative-frame redirect or same-frame navigation to a different origin; validate real APS inventory for intermediate or redirect origins before rollout. User-activated top navigation and popups remain governed by the sandbox tokens. - -A network-loaded HTTPS creative does not inherit its ancestor's CSP and can create further frames. To prevent it from using a publisher-origin descendant plus an executable publisher gadget to regain access to the top page, enabling APS appends a separate `Content-Security-Policy: frame-ancestors 'self'` policy to every Trusted Server response. Browsers require every ancestor to match, so publisher documents cannot load below the opaque container or third-party creative. This secure default also prevents the APS-enabled publisher from being embedded cross-origin; publishers that require trusted external embedders need a separately reviewed ancestor-allowlist feature before enabling APS. - -Trusted Server generates independent 128-bit nonces for the container and renderer. Once the inner renderer is ready, the container transfers a dedicated `MessagePort` between trusted top-page TSJS and the inner renderer. The complete descriptor travels only over that port; the inner renderer revalidates it, rejects the publisher origin, and requires the creative origin to equal the origin bound in the container CSP. The container URL contains only that exact origin, static renderer source, and nonces—never the response envelope, bid ID, price, or creative URL path. +```text +allow-forms +allow-pointer-lock +allow-popups +allow-popups-to-escape-sandbox +allow-scripts +allow-top-navigation-by-user-activation +``` -The inner document initializes the account-keyed APS queue and loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. A `renderer-ready` acknowledgement still means that Amazon's runner loaded, not that the final creative painted. Existing slot content remains until that acknowledgement. The query-free `GET /integrations/aps/renderer` response remains available with its original stricter CSP and legacy message shape for cached clients and rollback. +It deliberately omits `allow-same-origin`, so APS and bidder execution remains below an opaque-origin boundary. The renderer response repeats these restrictions with a CSP `sandbox` directive, preventing another embedding path from restoring publisher-origin execution by omitting the iframe attribute. Trusted Server generates a fresh 128-bit nonce, binds it in the iframe URL fragment before navigation, and requires the same one-time nonce in the parent message and renderer acknowledgement. Existing slot content is retained until the static renderer has accepted the descriptor and loaded the fixed runner. ### Direct `/auction` -The TSJS auction client validates the typed renderer descriptor and mounts the nested data renderer in the winning slot. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. +The TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. ### GAM and Universal Creative -For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program with a one-shot mount capability. That program asks trusted top-page TSJS to mount the nested data renderer as a sibling of the Universal Creative iframe, outside inherited GAM and Universal Creative sandbox restrictions. +For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program that creates the same opaque renderer iframe. -For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes both the renderer and mount capabilities once, and passes the APS bid ID separately to the Amazon runner. +For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes the capability once, and passes the APS bid ID separately to the Amazon runner. These paths do not fetch PBS Cache, fire generic APS win/billing beacons, or call `apstag.setDisplayBids()` for the Trusted Server winner. Publisher-owned native APS objects are otherwise left untouched. @@ -179,10 +182,10 @@ These paths do not fetch PBS Cache, fire generic APS win/billing beacons, or cal The publisher policy must permit the same-origin renderer route, for example: ```text -frame-src 'self' data: +frame-src 'self' ``` -The `data:` source is required for the bootstrap's self-navigation to the opaque container. APS also adds `frame-ancestors 'self'` as an independent response policy; do not override or remove that policy. Do not weaken or bypass the initial bootstrap sandbox. TSJS adds `allow-same-origin` only when navigating away from the publisher-origin bootstrap; both final data documents remain naturally opaque. The bootstrap response supplies the resource CSP for the fixed runner and HTTPS creative resources, while the container narrows `frame-src` to the validated creative origin. The same-origin bootstrap route inherits the publisher page scheme; use HTTPS in production. APS endpoints and third-party creative URLs always require HTTPS. +Do not add `allow-same-origin` to the outer renderer sandbox. The renderer endpoint supplies its own CSP for the fixed runner and HTTPS creative resources. The same-origin renderer route inherits the publisher page scheme; use HTTPS in production. APS endpoints and third-party creative URLs always require HTTPS. Before enabling script creatives, verify under the publisher's actual CSP that both iframe and script-tag creatives: @@ -232,8 +235,8 @@ Use fictional values in source-controlled configuration and fixtures. Supply con ### Winner targets but does not render -- Confirm `GET /integrations/aps/renderer?mode=data-bootstrap` returns HTML with its bootstrap CSP and `Referrer-Policy: no-referrer`. -- Confirm publisher CSP permits `frame-src 'self' data:`. +- Confirm `GET /integrations/aps/renderer` returns HTML with its CSP and `Referrer-Policy: no-referrer`. +- Confirm publisher CSP permits `frame-src 'self'`. - Confirm the GAM creative uses the supported Prebid Universal Creative bridge and the winning `hb_adid`. - For client-side `trustedServer` adapter auctions, confirm Prebid's `bidResponse` contains a generated `adId` and that the corresponding capability appears briefly in `window.tsjs.apsPrebidRenderers` before rendering. - Ensure no native APS path is trying to handle the same cohort. From 1a2d16c817ff6aa21052b4fbe175ffb3b20a418f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 10:19:43 +0530 Subject: [PATCH 332/395] Document comprehensive PR 928 review fixes --- ...-comprehensive-review-resolution-design.md | 237 ++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-19-pr928-comprehensive-review-resolution-design.md diff --git a/docs/superpowers/specs/2026-08-19-pr928-comprehensive-review-resolution-design.md b/docs/superpowers/specs/2026-08-19-pr928-comprehensive-review-resolution-design.md new file mode 100644 index 000000000..ee87ea9fd --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-pr928-comprehensive-review-resolution-design.md @@ -0,0 +1,237 @@ +# PR #928 Comprehensive Review Resolution + +**PR:** #928 +**Date:** 2026-08-19 +**Status:** Approved design + +## Problem + +PR #928 adds authenticated EC and EID diagnostic endpoints. Follow-up review +found three blocking correctness issues and six related quality gaps: + +1. Fastly's EC lookup still enters the mutating EC request/finalization + lifecycle, so a diagnostic GET can ingest browser cookies, mint identity + state, write a withdrawal tombstone, or trigger pull sync. +2. Startup authentication coverage checks only one lowercase-suffix EC ID. A + narrower handler regex can pass startup while valid mixed-case IDs fail + closed at runtime. +3. The API guide describes every response as JSON with `no-store`, although + the shared Basic-auth rejection is plaintext and has no cache header. +4. The EID preview omits configured sources whose UID list contains no value + accepted by the real ingestion path. +5. Core contains duplicate request-cookie extraction helpers. +6. Admin JSON responses lack `X-Content-Type-Options: nosniff`. +7. The tombstone field documentation omits typed deserialization failures. +8. New diagnostic routes lack explicit unauthenticated adapter regressions. +9. Operators are not warned that narrow pre-existing admin handler patterns + must expand to cover the new routes. + +## Goals + +- Make both Fastly diagnostic GET handlers read-only by construction. +- Detect common under-coverage of the full valid EC ID suffix alphabet during + settings finalization while retaining runtime fail-closed protection. +- Make the documented response contract accurately distinguish authentication + failures from successfully authenticated diagnostic-handler responses. +- Report every parsed EID source that ingestion drops, with an operator-useful + reason. +- Consolidate identical core cookie-header parsing without changing semantics. +- Apply browser-safe response headers consistently to admin diagnostic JSON. +- Pin the new routes' authentication behavior on every adapter. +- Document the intentional configuration compatibility impact. + +## Non-goals + +- Do not change Basic-auth middleware response bodies or headers. That shared + behavior predates the diagnostic endpoints and is outside this PR's scope. +- Do not attempt formal regex-language inclusion. Arbitrary configured regexes + make exhaustive proof impractical; runtime authentication remains the final + fail-closed invariant. +- Do not change live EID ingestion, partner matching, UID validation, or + deduplication behavior. +- Do not add EC lookup support to Axum, Cloudflare, or Spin. +- Do not introduce a test-only Fastly KV abstraction solely to spy on writes. +- Do not alter unrelated response builders or cookie parsing behavior. +- Do not push the branch, reply to GitHub threads, or resolve review + conversations as part of implementation. + +## Design + +### 1. Read-only Fastly diagnostic dispatch + +Move `AdminEcLookup` into the same `execute_named` early-dispatch branch as +`AdminEidsLookup`, before GPT diagnostic preparation, EC request-state +construction, and request filters. Construct the partner registry once, match +the requested diagnostic handler, and for EC lookup construct its KV identity +graph directly from settings, as the existing handler already does. Return the +handler response immediately without `attach_dispatch_extensions`. + +Keep exhaustive `run_named_route` arms for both diagnostic variants as +`unreachable!`, documenting that they must be handled before EC setup. Basic +authentication and normal outer response middleware continue to run because +they wrap `execute_named`. + +The regression uses an authenticated, browser-shaped EC request containing +`ts-ec`, `ts-eids`, and `sharedId` cookies plus device signals. It asserts a +successful diagnostic response has neither `EcFinalizeState` nor EC cookie +mutation headers. The Fastly entry point invokes EC finalization and all of its +KV writes only when `EcFinalizeState` is present, so absence tests the +production write gate without adding a test-only storage seam. Existing core +lookup tests continue to establish that the handler performs reads only. + +### 2. Dynamic authentication probe corpus + +Retain `Settings::ADMIN_ENDPOINTS` as the canonical operator-facing route list, +but map `/_ts/admin/ec/{id}` to a small fixed corpus of concrete valid IDs. The +corpus contains at least: + +- a lowercase-and-digit suffix such as `.abc123`; and +- a mixed-case-and-digit suffix such as `.Ab12Z9`. + +All probes use a valid 64-character lowercase hexadecimal hash. An endpoint is +covered only when every probe has a matching configured handler. Different +handlers may collectively cover the corpus because every matched handler still +requires Basic authentication. Placeholder-password validation classifies a +handler as protecting the dynamic admin route when it matches any probe, so a +narrow handler cannot escape credential-strength checks. + +Add a startup regression in which a lowercase-only suffix regex covers the +first probe but not the mixed-case probe. The configuration must be rejected +and continue to report the canonical `/_ts/admin/ec/{id}` template. Existing +runtime fail-closed authentication remains unchanged and protects valid IDs +outside the representative corpus. + +### 3. Accurate API and upgrade documentation + +Change the Admin Diagnostic Endpoints introduction to say that responses +produced after successful authentication are JSON with +`Cache-Control: no-store`. Explicitly note that missing or invalid credentials +use the shared plaintext `401 Unauthorized` challenge contract. + +Document the reason-tagged EID drop objects described below. Add an Unreleased +changelog entry stating that configurations which protected only the older key +management routes now fail startup and must broaden their authenticated handler +coverage to all `/_ts/admin` diagnostic routes. Recommend a namespace-wide +pattern such as `^/_ts/admin(?:/|$)` while preserving the existing warning +against accidentally protecting non-admin browser endpoints. + +### 4. Reason-tagged EID ingestion drops + +Replace `ingest.unmatched: string[]` with a list of objects containing: + +- `source`: the EID source string; and +- `reason`: `no_partner` or `no_valid_uid`. + +The production OpenRTB conversion intentionally removes structured entries +whose UID list becomes empty, so it cannot be the only diagnostic parse. Decode +the cookie once into the existing legacy-or-structured wire representation, +then return one shared analysis result with three derived views, without +changing live behavior: + +- the existing filtered `Vec` used by ingestion and returned in `eids`; +- a private diagnostic source view that retains non-empty source names even + when all supplied UIDs are empty or otherwise unusable; and +- the partner updates selected by the same lookup and first-valid-UID rules as + live ingestion. + +Classify retained `ts-eids` sources using the same partner lookup and valid-UID +predicate as live ingestion: + +- no configured partner for the source becomes `no_partner`; +- a configured partner with no non-empty UID within the ingestion size limit + becomes `no_valid_uid`; +- a configured partner with a valid UID is represented by the existing + deduplicated `matched` output and is not dropped. + +Group duplicate cookie entries by source for drop reporting. If any entry for +a configured source has a valid UID, emit no `no_valid_uid` drop for that +source; otherwise emit exactly one. An unconfigured source emits exactly one +`no_partner` drop. `sharedId` does not suppress a `ts-eids` drop because the +preview is explaining that source's own cookie input. + +Malformed `ts-eids` remains represented by `parse_error`, because there is no +parsed source to classify. `sharedId` behavior remains unchanged. This response +schema is safe to establish now because the endpoint is new in this unmerged +PR; the API guide and tests change in the same commit. + +Refactor decoding behind private helpers in the existing Prebid EID ingestion +module so the public production parser retains identical output. Add one +crate-visible analysis function used by the admin handler; make the production +update collector reuse the same analysis and extract only its updates. Expose +the UID-validity predicate within the crate as needed. This keeps one decode per +caller and prevents preview classification and live ingestion from drifting. + +### 5. Shared core cookie extraction + +Add a generic request-cookie value helper to the existing core `cookies` +module. It accepts `&http::Request` so both current core call sites can use +it regardless of body type, and preserves the existing behavior exactly: +inspect only the value selected by `headers().get(COOKIE)`; return `None` when +that selected value is absent or invalid UTF-8; otherwise split +semicolon-delimited pairs, trim whitespace, split only on the first `=`, and +return an owned value for the requested name. It does not scan later repeated +Cookie header values. + +Use it from `ec/admin.rs` and `auction/endpoints.rs`, deleting both local +copies. The Fastly adapter's separate helper operates on Fastly's platform +request type and remains local; forcing it through an incompatible abstraction +would expand scope without removing meaningful duplication. + +### 6. Diagnostic response hardening and field documentation + +Add `X-Content-Type-Options: nosniff` to the shared admin diagnostic +`json_response` builder. This covers EC lookup, EID preview, unsupported-adapter +responses, and local diagnostic fallback denials. Tests assert the header on +representative success and error responses. + +Update the `tombstone` field comment to state that it is absent when the body +cannot be parsed as JSON or deserialized as the typed `KvEntry` schema. No +runtime behavior changes. + +### 7. Cross-adapter authentication regressions + +Add one unauthenticated `GET /_ts/admin/ec` test to each adapter's established +route-test layer: Fastly, Axum, Cloudflare, and Spin. Each test asserts `401` +and the Basic `WWW-Authenticate` challenge. Keep existing authenticated route +tests unchanged so the pair establishes authentication-before-handler ordering +for both the Fastly implementation and portability adapters' `501` response. + +## Error Handling + +No new public failure mode is introduced. Partner-registry and KV-graph +construction errors continue through the existing `http_error` conversion. +Probe validation continues to return `TrustedServerError::Configuration` and +reports canonical route templates. EID preview classification is infallible +after cookie parsing. Shared cookie extraction intentionally ignores malformed +header encoding exactly as the removed helpers do. + +## Testing Strategy + +Implementation follows red-green-refactor, one review concern at a time: + +1. Add the cookie-bearing Fastly EC lookup regression, observe + `EcFinalizeState`, then early-dispatch the handler and verify its absence. +2. Add the lowercase-only auth-handler regression, observe startup success, + then require the mixed-case probe and verify rejection. +3. Add EID preview cases for `no_partner`, empty UID, and over-limit UID before + implementing the reason-tagged drop type and shared validity helper. +4. Add shared cookie-helper unit coverage, migrate the two core callers, and + run their focused tests. +5. Add `nosniff` assertions and the four adapter unauthenticated regressions. +6. Update API and changelog text and run documentation formatting. +7. Run the affected target suites, all repository-required format checks, and + target-matched clippy commands before claiming completion. + +## Expected Files + +- `crates/trusted-server-adapter-fastly/src/app.rs` +- `crates/trusted-server-adapter-axum/tests/routes.rs` +- `crates/trusted-server-adapter-cloudflare/tests/routes.rs` +- `crates/trusted-server-adapter-spin/tests/routes.rs` +- `crates/trusted-server-core/src/settings.rs` +- `crates/trusted-server-core/src/cookies.rs` +- `crates/trusted-server-core/src/auction/endpoints.rs` +- `crates/trusted-server-core/src/ec/admin.rs` +- `crates/trusted-server-core/src/ec/prebid_eids.rs` +- `docs/guide/api-reference.md` +- `CHANGELOG.md` From cf25efb7364275281fc4640bb3b4c4c6e3256cb6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 10:28:42 +0530 Subject: [PATCH 333/395] Plan comprehensive PR 928 review fixes --- ...9-pr928-comprehensive-review-resolution.md | 487 ++++++++++++++++++ 1 file changed, 487 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-19-pr928-comprehensive-review-resolution.md diff --git a/docs/superpowers/plans/2026-08-19-pr928-comprehensive-review-resolution.md b/docs/superpowers/plans/2026-08-19-pr928-comprehensive-review-resolution.md new file mode 100644 index 000000000..9095e7853 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-pr928-comprehensive-review-resolution.md @@ -0,0 +1,487 @@ +# PR #928 Comprehensive Review Resolution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve all actionable PR #928 review findings while preserving existing authentication, cookie-ingestion, and adapter behavior outside the new admin diagnostics. + +**Architecture:** Fastly handles both diagnostic routes before the EC lifecycle; core settings checks the parameterized admin route against a representative valid-ID corpus; core EID parsing exposes a source-preserving diagnostic view alongside unchanged production output. Shared core cookie parsing and admin JSON headers remove duplication and response drift, while adapter regressions and documentation pin the external contract. + +**Tech Stack:** Rust 2024, `http`, `serde`, `serde_json`, `error-stack`, EdgeZero adapter routers, Fastly/Viceroy, Markdown/VitePress. + +**Design spec:** `docs/superpowers/specs/2026-08-19-pr928-comprehensive-review-resolution-design.md` + +--- + +## File Map + +- Modify `crates/trusted-server-adapter-fastly/src/app.rs`: early EC diagnostic dispatch and Fastly authentication/finalization regressions. +- Modify `crates/trusted-server-core/src/settings.rs`: dynamic-route authentication probe corpus and settings regression. +- Modify `crates/trusted-server-core/src/ec/prebid_eids.rs`: source-preserving diagnostic parse view and shared UID-validity rule. +- Modify `crates/trusted-server-core/src/ec/admin.rs`: reason-tagged EID drops, shared cookie helper use, `nosniff`, and unit tests. +- Modify `crates/trusted-server-core/src/cookies.rs`: generic request-cookie extraction helper and focused tests. +- Modify `crates/trusted-server-core/src/auction/endpoints.rs`: use the shared cookie helper. +- Modify `crates/trusted-server-adapter-axum/tests/routes.rs`: unauthenticated diagnostic regression. +- Modify `crates/trusted-server-adapter-cloudflare/tests/routes.rs`: unauthenticated diagnostic regression. +- Modify `crates/trusted-server-adapter-spin/tests/routes.rs`: unauthenticated diagnostic regression. +- Modify `docs/guide/api-reference.md`: accurate authentication response contract and reason-tagged preview schema. +- Modify `CHANGELOG.md`: Unreleased configuration compatibility warning. + +No dependency, public configuration schema, or test-only production seam is added. + +### Task 1: Keep Fastly EC diagnostics outside the mutating lifecycle + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs:527-610` +- Test: `crates/trusted-server-adapter-fastly/src/app.rs:2200-2250` + +- [ ] **Step 1: Add the failing cookie-bearing EC diagnostic regression** + +Add `admin_ec_diagnostic_skips_ec_finalization` next to the EIDs equivalent. +Build an authenticated `GET /_ts/admin/ec/{valid_id}` request carrying +`ts-ec`, base64-encoded `ts-eids`, and `sharedId`, plus browser-shaped +`DeviceSignals`. Assert the response has no `EcFinalizeState` and no +`Set-Cookie` header. + +- [ ] **Step 2: Run the focused test and verify RED** + +```bash +cargo test-fastly admin_ec_diagnostic_skips_ec_finalization +``` + +Expected: FAIL because the current response carries `EcFinalizeState`. + +- [ ] **Step 3: Early-dispatch both diagnostic handlers** + +Change the current `AdminEidsLookup` early branch to match both variants: + +```rust +if matches!( + handler, + NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup +) { + let response = PartnerRegistry::from_config(&state.settings.ec.partners) + .and_then(|registry| match handler { + NamedRouteHandler::AdminEcLookup => { + let kv = crate::maybe_identity_graph(&state.settings); + handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) + } + NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), + _ => unreachable!("admin diagnostics should match an early-dispatch handler"), + }) + .unwrap_or_else(|error| http_error(&error)); + return Ok(response); +} +``` + +Replace the later `AdminEcLookup` implementation arm with `unreachable!`, like +the EIDs arm. Update comments to describe both diagnostics. + +- [ ] **Step 4: Run both finalization regressions and verify GREEN** + +```bash +cargo test-fastly diagnostic_skips_ec_finalization +``` + +Expected: both EC and EIDs tests PASS. + +- [ ] **Step 5: Commit Task 1** + +```bash +git add crates/trusted-server-adapter-fastly/src/app.rs +git commit -m "Keep Fastly admin EC lookups read only" +``` + +### Task 2: Validate dynamic admin authentication with a suffix corpus + +**Files:** + +- Modify: `crates/trusted-server-core/src/settings.rs:2200-2300` +- Test: `crates/trusted-server-core/src/settings.rs:4960-5035` + +- [ ] **Step 1: Add a failing mixed-case coverage regression** + +Add `from_toml_rejects_lowercase_only_dynamic_admin_ec_auth_coverage`. Configure +the static admin routes with strong credentials and the parameterized route +with: + +```toml +[[handlers]] +path = "^/_ts/admin/ec/[a-f0-9]{64}[.][a-z0-9]{6}$" +username = "admin" +password = "strong-test-password" +``` + +Assert `Settings::from_toml` returns a configuration error naming +`/_ts/admin/ec/{id}`. + +- [ ] **Step 2: Run the regression and verify RED** + +```bash +cargo test-fastly from_toml_rejects_lowercase_only_dynamic_admin_ec_auth_coverage +``` + +Expected: FAIL because `.abc123` is the only probe and the settings load. + +- [ ] **Step 3: Replace the single probe with a fixed corpus** + +Define lowercase and mixed-case concrete paths, and make the mapping return a +slice: + +```rust +const ADMIN_EC_ID_AUTH_PROBES: &[&str] = &[ + concat!("/_ts/admin/ec/", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", ".abc123"), + concat!("/_ts/admin/ec/", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", ".Ab12Z9"), +]; + +fn admin_auth_probes(path: &'static str) -> &'static [&'static str] { + match path { + "/_ts/admin/ec/{id}" => Self::ADMIN_EC_ID_AUTH_PROBES, + path => core::slice::from_ref(&path), + } +} +``` + +If `core::slice::from_ref` cannot produce the required static lifetime for the +match binding, use static one-element probe arrays for the non-parameterized +routes rather than allocating. + +In `uncovered_admin_endpoints`, require every probe to match at least one +handler. In `validate_admin_handler_passwords`, classify a handler as admin +when it matches any probe for any canonical endpoint. Preserve canonical +template reporting. + +- [ ] **Step 4: Run the focused and existing coverage tests** + +```bash +cargo test-fastly dynamic_admin_ec_auth_coverage +cargo test-fastly literal_parameter_template_auth_coverage +cargo test-fastly placeholder_password_for_concrete_admin_ec_handler +cargo test-fastly uncovered_admin_endpoints +``` + +Expected: PASS. + +- [ ] **Step 5: Commit Task 2** + +```bash +git add crates/trusted-server-core/src/settings.rs +git commit -m "Validate mixed-case admin EC auth coverage" +``` + +### Task 3: Report reason-tagged EID ingestion drops + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/prebid_eids.rs:25-110,178-230,275-340` +- Modify: `crates/trusted-server-core/src/ec/admin.rs:410-515` +- Test: `crates/trusted-server-core/src/ec/prebid_eids.rs:400-620` +- Test: `crates/trusted-server-core/src/ec/admin.rs:1075-1185` + +- [ ] **Step 1: Add failing admin preview regressions** + +Update the existing unmatched assertion to expect: + +```json +{"source":"unknown.example","reason":"no_partner"} +``` + +Add a test whose configured source has only whitespace/empty and over-limit +UID candidates; expect one `no_valid_uid` drop. Add a duplicate-source test +where one entry has no valid UID and a later entry has a valid UID; expect the +source in `matched` and no drop. + +- [ ] **Step 2: Run the focused tests and verify RED** + +```bash +cargo test-fastly eids_lookup_ +``` + +Expected: existing string-shaped unmatched output fails and invalid-only +sources are absent. + +- [ ] **Step 3: Refactor cookie decoding without changing production output** + +In `prebid_eids.rs`, introduce a private decoded wire enum holding +`Vec` or `Vec`. Move size/base64/JSON +selection into one decoder. Keep `parse_prebid_eids_cookie` public and map the +decoded wire representation through the existing conversion functions so all +current parser tests remain unchanged. + +Add a crate-visible `PrebidEidAnalysis` and analysis function that decode once +and derive all data the admin handler needs: the filtered `Vec`, retained +diagnostic sources with raw UID strings, and `Vec` selected by +the live partner/UID rules. Make `collect_prebid_eid_updates` call the same +analysis function and extract only `updates`. Add or expose a crate-visible +predicate implementing the existing live rule: + +```rust +pub(crate) fn is_valid_eid_uid(uid: &str) -> bool { + !uid.trim().is_empty() && !eid_id_exceeds_size_limit(uid) +} +``` + +Make `first_valid_uid` call this predicate. + +- [ ] **Step 4: Implement deterministic drop classification** + +In `admin.rs`, replace `Vec` with: + +```rust +#[derive(Debug, Serialize)] +struct DroppedEidSource { + source: String, + reason: DroppedEidReason, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "snake_case")] +enum DroppedEidReason { + NoPartner, + NoValidUid, +} +``` + +Group diagnostic sources in a `BTreeMap` for stable output. Emit one +`NoPartner` for an unconfigured source. For a configured source, emit one +`NoValidUid` only when no entry contains a UID satisfying +`is_valid_eid_uid`. Do not let `sharedId` suppress a `ts-eids` drop. Preserve +the existing matched-update collection and deduplication. + +Replace the admin handler's separate `parse_prebid_eids_cookie` and +`collect_prebid_eid_updates` calls with one `analyze_prebid_eids_cookie` call. +On success, move its filtered EIDs into the response, classify its diagnostic +sources, and extend the matched-update list from its updates. On failure, set +the existing `parse_error` and produce no EIDs, drops, or Prebid updates. + +- [ ] **Step 5: Run parser, preview, and ingestion tests** + +```bash +cargo test-fastly prebid_eids +cargo test-fastly eids_lookup_ +``` + +Expected: PASS, including unchanged live-ingestion cases. + +- [ ] **Step 6: Commit Task 3** + +```bash +git add crates/trusted-server-core/src/ec/prebid_eids.rs crates/trusted-server-core/src/ec/admin.rs +git commit -m "Explain dropped admin EID preview sources" +``` + +### Task 4: Consolidate core request-cookie extraction + +**Files:** + +- Modify: `crates/trusted-server-core/src/cookies.rs:1-120` +- Modify: `crates/trusted-server-core/src/ec/admin.rs:20-45,225-245,445-525` +- Modify: `crates/trusted-server-core/src/auction/endpoints.rs:1-35,240-255,400-430` +- Test: `crates/trusted-server-core/src/cookies.rs` + +- [ ] **Step 1: Add focused helper tests** + +Add tests for missing header, whitespace trimming, a value containing `=`, and +multiple cookie pairs. Add a request with two Cookie header values where the +selected `headers().get` value is invalid UTF-8 and assert `None`, pinning the +old helper semantics rather than scanning later values. + +- [ ] **Step 2: Run the focused test and verify RED** + +```bash +cargo test-fastly extract_cookie_value +``` + +Expected: compilation FAIL because the shared helper does not exist. + +- [ ] **Step 3: Add the generic shared helper** + +Add a documented crate-public function in `cookies.rs`: + +```rust +#[must_use] +pub fn extract_cookie_value(req: &Request, name: &str) -> Option { + let cookie_header = req.headers().get(header::COOKIE)?.to_str().ok()?; + cookie_header.split(';').find_map(|pair| { + let (key, value) = pair.trim().split_once('=')?; + (key.trim() == name).then(|| value.trim().to_owned()) + }) +} +``` + +- [ ] **Step 4: Migrate both core callers and delete local copies** + +Import `crate::cookies::extract_cookie_value` in `ec/admin.rs` and +`auction/endpoints.rs`. Remove their byte-identical local helpers and any +imports made unused. + +- [ ] **Step 5: Run focused caller tests** + +```bash +cargo test-fastly extract_cookie_value +cargo test-fastly eids_lookup_ +cargo test-fastly auction +``` + +Expected: PASS. + +- [ ] **Step 6: Commit Task 4** + +```bash +git add crates/trusted-server-core/src/cookies.rs crates/trusted-server-core/src/ec/admin.rs crates/trusted-server-core/src/auction/endpoints.rs +git commit -m "Share core request cookie extraction" +``` + +### Task 5: Harden diagnostic JSON and pin adapter authentication + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/admin.rs:90-130,530-550` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs:1800-1875` +- Modify: `crates/trusted-server-adapter-axum/tests/routes.rs:245-325` +- Modify: `crates/trusted-server-adapter-cloudflare/tests/routes.rs:275-330` +- Modify: `crates/trusted-server-adapter-spin/tests/routes.rs:105-165` + +- [ ] **Step 1: Add failing `nosniff` response assertions** + +In core admin tests, assert representative success and JSON error responses +contain `X-Content-Type-Options: nosniff`. + +- [ ] **Step 2: Run the focused test and verify RED** + +```bash +cargo test-fastly admin_ec_lookup +``` + +Expected: FAIL because the header is absent. + +- [ ] **Step 3: Add the shared response header and correct field docs** + +Add `.header(header::X_CONTENT_TYPE_OPTIONS, "nosniff")` to +`json_response`. Update `tombstone` documentation to say it is absent when the +body fails JSON parsing or typed `KvEntry` deserialization. + +- [ ] **Step 4: Add one new-route unauthenticated test per adapter** + +For Fastly, Axum, Cloudflare, and Spin, send `GET /_ts/admin/ec` without an +Authorization header. Assert `401 Unauthorized` and the existing Basic +`WWW-Authenticate` realm. Place each test next to the adapter's authenticated +EC diagnostic test and reuse its established router/service helper. + +- [ ] **Step 5: Run each adapter's focused authentication test** + +```bash +cargo test-fastly admin_ec_route_without_credentials +cargo test-axum admin_ec_route_without_credentials +cargo test-cloudflare admin_ec_route_without_credentials +cargo test-spin admin_ec_route_without_credentials +``` + +Expected: PASS. + +- [ ] **Step 6: Commit Task 5** + +```bash +git add crates/trusted-server-core/src/ec/admin.rs crates/trusted-server-adapter-fastly/src/app.rs crates/trusted-server-adapter-axum/tests/routes.rs crates/trusted-server-adapter-cloudflare/tests/routes.rs crates/trusted-server-adapter-spin/tests/routes.rs +git commit -m "Harden admin diagnostic responses" +``` + +### Task 6: Correct operator-facing documentation + +**Files:** + +- Modify: `docs/guide/api-reference.md:580-640` +- Modify: `CHANGELOG.md:8-25` + +- [ ] **Step 1: Qualify authentication response behavior** + +State that successfully authenticated diagnostic-handler responses are JSON +with `Cache-Control: no-store`, while missing or invalid credentials receive +the shared plaintext `401 Unauthorized` Basic challenge. + +- [ ] **Step 2: Document reason-tagged EID drops** + +Describe `ingest.unmatched` entries as `{source, reason}` objects and define +`no_partner` and `no_valid_uid`. Include a compact example covering one match +and one drop. + +- [ ] **Step 3: Add the Unreleased compatibility warning** + +Under `CHANGELOG.md`'s Unreleased Changed section, state that startup now +requires authenticated handler coverage for the EC/EID diagnostics in addition +to key management. Tell operators with narrow key-only patterns to broaden +coverage before deploying, preferably to `^/_ts/admin(?:/|$)`. + +- [ ] **Step 4: Format documentation** + +```bash +cd docs && npm run format +``` + +Expected: formatter exits 0 with only intended Markdown changes. + +- [ ] **Step 5: Commit Task 6** + +```bash +git add docs/guide/api-reference.md CHANGELOG.md +git commit -m "Clarify admin diagnostics contracts" +``` + +### Task 7: Full verification + +**Files:** + +- Verify all modified files. + +- [ ] **Step 1: Check formatting** + +```bash +cargo fmt --all -- --check +``` + +Expected: PASS. + +- [ ] **Step 2: Run adapter and core test suites** + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: all PASS. + +- [ ] **Step 3: Run target-matched clippy checks** + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: all PASS with warnings denied by repository aliases. + +- [ ] **Step 4: Verify documentation and diff hygiene** + +```bash +cd docs && npm run format +git diff --check +git status --short +``` + +Expected: formatter and diff check PASS; status shows only intentional plan or +implementation state. + +- [ ] **Step 5: Review the branch diff against the PR base** + +```bash +git diff --stat origin/main...HEAD +git diff origin/main...HEAD -- crates/trusted-server-adapter-fastly/src/app.rs crates/trusted-server-core/src/settings.rs crates/trusted-server-core/src/ec/admin.rs crates/trusted-server-core/src/ec/prebid_eids.rs crates/trusted-server-core/src/cookies.rs crates/trusted-server-core/src/auction/endpoints.rs docs/guide/api-reference.md CHANGELOG.md +``` + +Expected: every change maps to the approved spec; no unrelated refactor or +behavior change is present. From f3776a9a7c3849e60ff6b32318db973ab1770a11 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 10:31:10 +0530 Subject: [PATCH 334/395] Keep Fastly admin EC lookups read only --- .../trusted-server-adapter-fastly/src/app.rs | 85 +++++++++++++++---- 1 file changed, 67 insertions(+), 18 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 9a8ea1a3a..36ea045de 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -52,8 +52,8 @@ //! `route_request` (tracked in issue #495): //! //! - [`build_ec_request_state`] runs before every dispatched route (except -//! batch-sync, which uses Bearer auth, and the read-only admin EIDs -//! diagnostic) and reproduces the legacy +//! batch-sync, which uses Bearer auth, and the read-only admin diagnostics) +//! and reproduces the legacy //! pre-routing prelude: device signals, bot gate, `ts-eids`/`sharedid` //! cookie capture, geo lookup, [`EcContext`] creation, and KV-graph gating. //! - `handle_auction` and integration proxy dispatch receive the same @@ -532,13 +532,27 @@ async fn execute_named( return Ok(run_batch_sync(&state, &services, req)); } - // This diagnostic only previews request cookies. Running the normal EC - // lifecycle would attach finalization state and could ingest those cookies - // into KV after the handler returns, violating the endpoint's read-only - // contract. - if matches!(handler, NamedRouteHandler::AdminEidsLookup) { + // These diagnostics are read-only. Running the normal EC lifecycle would + // attach finalization state and could ingest request cookies into KV after + // the handler returns, violating that contract. + if matches!( + handler, + NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup + ) { let response = PartnerRegistry::from_config(&state.settings.ec.partners) - .and_then(|registry| handle_admin_eids_lookup(®istry, &req)) + .and_then(|registry| match handler { + NamedRouteHandler::AdminEcLookup => { + // Deliberately do not use an EC request-state graph: that + // copy is bot-gated, while operators use curl for this + // authenticated diagnostic. + let kv = crate::maybe_identity_graph(&state.settings); + handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) + } + NamedRouteHandler::AdminEidsLookup => { + handle_admin_eids_lookup(®istry, &req) + } + _ => unreachable!("admin diagnostics should use early dispatch"), + }) .unwrap_or_else(|error| http_error(&error)); return Ok(response); } @@ -592,16 +606,8 @@ async fn run_named_route( } NamedRouteHandler::RotateKey => handle_rotate_key(&state.settings, services, req), NamedRouteHandler::DeactivateKey => handle_deactivate_key(&state.settings, services, req), - NamedRouteHandler::AdminEcLookup => { - // Deliberately NOT `ec.kv_graph`: that copy is bot-gated (None for - // non-browser clients), and operators hit this auth-gated endpoint - // with curl. Build the graph directly from settings instead. - let kv = crate::maybe_identity_graph(&state.settings); - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; - handle_admin_ec_lookup(kv.as_ref(), &partner_registry, &req) - } - NamedRouteHandler::AdminEidsLookup => { - unreachable!("admin EIDs lookup should be handled before EC setup") + NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup => { + unreachable!("admin diagnostics should be handled before EC setup") } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::BatchSync => { @@ -2235,6 +2241,49 @@ mod tests { ); } + #[test] + fn admin_ec_diagnostic_skips_ec_finalization() { + let router = test_router(); + let ec_id = format!("{}.abc123", "a".repeat(64)); + let eids = serde_json::json!([{ + "source": "example.com", + "uids": [{ "id": "example-uid", "atype": 1 }] + }]); + let eids_cookie = base64::engine::general_purpose::STANDARD.encode(eids.to_string()); + let mut request = request_builder() + .method(Method::GET) + .uri(format!( + "https://test-publisher.com/_ts/admin/ec/{ec_id}" + )) + .header(header::AUTHORIZATION, "Basic YWRtaW46YWRtaW4tcGFzcw==") + .header( + header::COOKIE, + format!("ts-ec={ec_id}; ts-eids={eids_cookie}; sharedId=example-shared-id"), + ) + .body(Body::empty()) + .expect("should build authenticated EC diagnostic request"); + request.extensions_mut().insert(DeviceSignals::derive( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + Some("t13d1516h2_8daaf6152771_b186095e22b6"), + Some("1:65536;2:0;4:6291456;6:262144"), + )); + + let response = route(&router, request); + + assert!( + response + .extensions() + .get::() + .is_none(), + "admin EC diagnostics should not attach EC finalization state" + ); + assert!( + response.headers().get(header::SET_COOKIE).is_none(), + "admin EC diagnostics should not mutate the EC cookie" + ); + } + #[test] fn dispatch_head_on_named_get_route_falls_through_to_publisher_fallback() { // Regression guard: HEAD /first-party/proxy must reach the publisher From de61c43f13cd6b8f15ad6ea3fd09dcc5385b15d6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 10:33:54 +0530 Subject: [PATCH 335/395] Validate mixed-case admin EC auth coverage --- crates/trusted-server-core/src/settings.rs | 81 ++++++++++++++++------ 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 8ba010901..de16aa758 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2207,8 +2207,8 @@ impl Settings { /// admin routes to `crates/trusted-server-adapter-fastly/src/app.rs`. /// /// The `/_ts/admin/ec/{id}` entry is the canonical router pattern. Handler - /// coverage is checked against a representative concrete EC ID via - /// [`admin_auth_probe`](Self::admin_auth_probe), while validation errors + /// coverage is checked against representative concrete EC IDs via + /// [`admin_auth_probes`](Self::admin_auth_probes), while validation errors /// continue to report this operator-facing route template. pub(crate) const ADMIN_ENDPOINTS: &[&str] = &[ "/_ts/admin/keys/rotate", @@ -2218,16 +2218,23 @@ impl Settings { "/_ts/admin/eids", ]; - const ADMIN_EC_ID_AUTH_PROBE: &str = concat!( - "/_ts/admin/ec/", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - ".abc123", - ); + const ADMIN_EC_ID_AUTH_PROBES: [&str; 2] = [ + concat!( + "/_ts/admin/ec/", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ".abc123", + ), + concat!( + "/_ts/admin/ec/", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ".Ab12Z9", + ), + ]; - fn admin_auth_probe(path: &'static str) -> &'static str { + fn admin_auth_probes(path: &'static str) -> [&'static str; 2] { match path { - "/_ts/admin/ec/{id}" => Self::ADMIN_EC_ID_AUTH_PROBE, - path => path, + "/_ts/admin/ec/{id}" => Self::ADMIN_EC_ID_AUTH_PROBES, + path => [path, path], } } @@ -2245,12 +2252,16 @@ impl Settings { ) -> Result, Report> { let mut uncovered = Vec::new(); for &path in Self::ADMIN_ENDPOINTS { - let mut covered = false; - for h in &self.handlers { - if h.matches_path(Self::admin_auth_probe(path))? { - covered = true; - break; + let mut covered = true; + for probe in Self::admin_auth_probes(path) { + let mut probe_covered = false; + for handler in &self.handlers { + if handler.matches_path(probe)? { + probe_covered = true; + break; + } } + covered &= probe_covered; } if !covered { uncovered.push(path); @@ -2284,10 +2295,15 @@ impl Settings { for handler in &self.handlers { let covers_admin = Self::ADMIN_ENDPOINTS .iter() - .try_fold(false, |covered, path| { - handler - .matches_path(Self::admin_auth_probe(path)) - .map(|matches| covered || matches) + .try_fold(false, |covers_any_endpoint, path| { + Self::admin_auth_probes(path).iter().try_fold( + covers_any_endpoint, + |covers_any_probe, probe| { + handler + .matches_path(probe) + .map(|matches| covers_any_probe || matches) + }, + ) })?; if covers_admin && is_admin_placeholder_password(handler.password.expose()) { @@ -4982,7 +4998,7 @@ origin_host_header_overide = "www.example.com""#, } #[test] - fn from_toml_rejects_placeholder_password_for_concrete_admin_ec_handler() { + fn from_toml_rejects_lowercase_only_dynamic_admin_ec_auth_coverage() { let toml_str = crate_test_settings_str().replace( r#"path = "^/_ts/admin" username = "admin" @@ -4994,6 +5010,31 @@ origin_host_header_overide = "www.example.com""#, [[handlers]] path = "^/_ts/admin/ec/[a-f0-9]{64}[.][a-z0-9]{6}$" username = "admin" + password = "strong-test-password""#, + ); + + let error = Settings::from_toml(&toml_str) + .expect_err("should reject lowercase-only dynamic EC auth coverage"); + let message = format!("{error:?}"); + assert!( + message.contains("/_ts/admin/ec/{id}"), + "should identify the mixed-case EC route as uncovered, got: {message}" + ); + } + + #[test] + fn from_toml_rejects_placeholder_password_for_concrete_admin_ec_handler() { + let toml_str = crate_test_settings_str().replace( + r#"path = "^/_ts/admin" + username = "admin" + password = "admin-pass""#, + r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" + username = "admin" + password = "strong-test-password" + + [[handlers]] + path = "^/_ts/admin/ec/[a-f0-9]{64}[.][A-Za-z0-9]{6}$" + username = "admin" password = "change-me-admin-password""#, ); From 8649b43e9d0ef13ee78e78581a87baed0e4a88da Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 10:37:42 +0530 Subject: [PATCH 336/395] Explain dropped admin EID preview sources --- .../trusted-server-adapter-fastly/src/app.rs | 8 +- crates/trusted-server-core/src/ec/admin.rs | 142 +++++++++++++++--- .../trusted-server-core/src/ec/prebid_eids.rs | 89 ++++++++++- crates/trusted-server-core/src/settings.rs | 25 +-- 4 files changed, 217 insertions(+), 47 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 36ea045de..ca0f528a2 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -548,9 +548,7 @@ async fn execute_named( let kv = crate::maybe_identity_graph(&state.settings); handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) } - NamedRouteHandler::AdminEidsLookup => { - handle_admin_eids_lookup(®istry, &req) - } + NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), _ => unreachable!("admin diagnostics should use early dispatch"), }) .unwrap_or_else(|error| http_error(&error)); @@ -2252,9 +2250,7 @@ mod tests { let eids_cookie = base64::engine::general_purpose::STANDARD.encode(eids.to_string()); let mut request = request_builder() .method(Method::GET) - .uri(format!( - "https://test-publisher.com/_ts/admin/ec/{ec_id}" - )) + .uri(format!("https://test-publisher.com/_ts/admin/ec/{ec_id}")) .header(header::AUTHORIZATION, "Basic YWRtaW46YWRtaW4tcGFzcw==") .header( header::COOKIE, diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 46170c53a..c5b5e2d59 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -16,6 +16,8 @@ //! auth-gated and operator-facing, responses intentionally include full //! internal detail (raw consent strings, partner UIDs, parse errors). +use std::collections::BTreeMap; + use http::{HeaderValue, Method, Request, Response, StatusCode, header}; use serde::Serialize; use serde_json::Value as JsonValue; @@ -34,8 +36,7 @@ use super::kv_backend::EcKvLookup; use super::kv_types::{KvEntry, KvMetadata}; use super::log_id; use super::prebid_eids::{ - collect_prebid_eid_updates, collect_sharedid_update, dedupe_partner_updates, - parse_prebid_eids_cookie, + analyze_prebid_eids_cookie, collect_sharedid_update, dedupe_partner_updates, is_valid_eid_uid, }; use super::registry::PartnerRegistry; @@ -421,8 +422,8 @@ struct IngestPreview { /// Cookie sources matched to a configured partner, with the UID that /// would be stored (deduplicated exactly like the ingestion path). matched: Vec, - /// `ts-eids` sources with no configured partner; dropped on ingestion. - unmatched: Vec, + /// `ts-eids` sources dropped on ingestion, with the reason. + unmatched: Vec, } /// A cookie-derived partner UID that ingestion would store. @@ -434,6 +435,21 @@ struct MatchedPartnerId { uid: String, } +#[derive(Debug, Serialize)] +struct DroppedEidSource { + /// EID source from the cookie. + source: String, + /// Why ingestion would drop the source. + reason: DroppedEidReason, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "snake_case")] +enum DroppedEidReason { + NoPartner, + NoValidUid, +} + /// Handles `GET /_ts/admin/eids`. /// /// Echoes the request's `ts-eids` and `sharedId` cookies: the parsed EID @@ -455,13 +471,20 @@ pub fn handle_admin_eids_lookup( let eids_cookie = extract_cookie_value(req, COOKIE_TS_EIDS); let sharedid_cookie = extract_cookie_value(req, COOKIE_SHAREDID); - let (eids, parse_error) = match &eids_cookie { - None => (None, None), - Some(value) => match parse_prebid_eids_cookie(value) { - Ok(parsed) => (Some(parsed), None), + let (eids, parse_error, diagnostic_sources, mut updates) = match &eids_cookie { + None => (None, None, Vec::new(), Vec::new()), + Some(value) => match analyze_prebid_eids_cookie(value, registry) { + Ok(analysis) => ( + Some(analysis.eids), + None, + analysis.diagnostic_sources, + analysis.updates, + ), Err(error) => ( None, Some(format!("failed to parse ts-eids cookie: {error}")), + Vec::new(), + Vec::new(), ), }, }; @@ -469,10 +492,6 @@ pub fn handle_admin_eids_lookup( // Mirror the ingestion path (`ingest_eid_cookies`): collect matches from // both cookies, then dedupe the same way so the preview reports exactly // what a navigation would store. - let mut updates = Vec::new(); - if let Some(value) = &eids_cookie { - updates.extend(collect_prebid_eid_updates(value, registry)); - } if let Some(value) = &sharedid_cookie && let Some(update) = collect_sharedid_update(value, registry) { @@ -486,16 +505,30 @@ pub fn handle_admin_eids_lookup( }) .collect(); - let unmatched = eids - .as_ref() - .map(|parsed| { - parsed - .iter() - .filter(|eid| registry.find_by_source_domain(&eid.source).is_none()) - .map(|eid| eid.source.clone()) - .collect() + let mut source_has_valid_uid = BTreeMap::new(); + for diagnostic_source in diagnostic_sources { + let has_valid_uid = diagnostic_source + .uids + .iter() + .any(|uid| is_valid_eid_uid(uid)); + source_has_valid_uid + .entry(diagnostic_source.source) + .and_modify(|source_has_valid_uid| *source_has_valid_uid |= has_valid_uid) + .or_insert(has_valid_uid); + } + let unmatched = source_has_valid_uid + .into_iter() + .filter_map(|(source, has_valid_uid)| { + let reason = if registry.find_by_source_domain(&source).is_none() { + DroppedEidReason::NoPartner + } else if !has_valid_uid { + DroppedEidReason::NoValidUid + } else { + return None; + }; + Some(DroppedEidSource { source, reason }) }) - .unwrap_or_default(); + .collect(); let payload = AdminEidsResponse { cookie_present: eids_cookie.is_some(), @@ -1134,7 +1167,72 @@ mod tests { .as_array() .expect("should have unmatched list"); assert_eq!(unmatched.len(), 1, "should report the unregistered source"); - assert_eq!(unmatched[0], "unknown.example"); + assert_eq!(unmatched[0]["source"], "unknown.example"); + assert_eq!(unmatched[0]["reason"], "no_partner"); + } + + #[test] + fn eids_lookup_reports_configured_source_without_valid_uid() { + let oversized_uid = "x".repeat(513); + let cookie = eids_cookie_for(&serde_json::json!([{ + "source": "bidstream.example", + "uids": [ + { "id": "", "atype": 1 }, + { "id": " ", "atype": 1 }, + { "id": oversized_uid, "atype": 1 } + ] + }])); + let req = get_request_with_cookie("/_ts/admin/eids", &format!("ts-eids={cookie}")); + + let response = + handle_admin_eids_lookup(&test_registry(), &req).expect("should handle eids lookup"); + let json = response_json(response); + + assert!( + json["ingest"]["matched"] + .as_array() + .expect("should have matched list") + .is_empty(), + "invalid UIDs should not be matched" + ); + let unmatched = json["ingest"]["unmatched"] + .as_array() + .expect("should have unmatched list"); + assert_eq!(unmatched.len(), 1, "should report one dropped source"); + assert_eq!(unmatched[0]["source"], "bidstream.example"); + assert_eq!(unmatched[0]["reason"], "no_valid_uid"); + } + + #[test] + fn eids_lookup_does_not_drop_duplicate_source_with_valid_uid() { + let cookie = eids_cookie_for(&serde_json::json!([ + { + "source": "bidstream.example", + "uids": [{ "id": " ", "atype": 1 }] + }, + { + "source": "bidstream.example", + "uids": [{ "id": "uid-valid", "atype": 1 }] + } + ])); + let req = get_request_with_cookie("/_ts/admin/eids", &format!("ts-eids={cookie}")); + + let response = + handle_admin_eids_lookup(&test_registry(), &req).expect("should handle eids lookup"); + let json = response_json(response); + + let matched = json["ingest"]["matched"] + .as_array() + .expect("should have matched list"); + assert_eq!(matched.len(), 1, "should match the valid duplicate source"); + assert_eq!(matched[0]["uid"], "uid-valid"); + assert!( + json["ingest"]["unmatched"] + .as_array() + .expect("should have unmatched list") + .is_empty(), + "a valid duplicate should suppress no_valid_uid" + ); } #[test] diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 0d7167e65..4a1a8d156 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -49,6 +49,22 @@ struct StructuredCookieUid { ext: Option, } +enum DecodedCookieEids { + Legacy(Vec), + Structured(Vec), +} + +pub(crate) struct DiagnosticEidSource { + pub(crate) source: String, + pub(crate) uids: Vec, +} + +pub(crate) struct PrebidEidAnalysis { + pub(crate) eids: Vec, + pub(crate) diagnostic_sources: Vec, + pub(crate) updates: Vec, +} + trait PartnerIdBulkWriter { fn upsert_partner_ids( &self, @@ -77,6 +93,10 @@ impl PartnerIdBulkWriter for KvIdentityGraph { /// Returns an error when the cookie exceeds the raw size limit, is not valid /// base64, or does not contain either supported JSON payload shape. pub fn parse_prebid_eids_cookie(cookie_value: &str) -> Result, String> { + decode_prebid_eids_cookie(cookie_value).map(DecodedCookieEids::into_openrtb) +} + +fn decode_prebid_eids_cookie(cookie_value: &str) -> Result { if eids_cookie_exceeds_size_limit(cookie_value) { return Err(format!( "ts-eids cookie too large ({} bytes)", @@ -89,12 +109,42 @@ pub fn parse_prebid_eids_cookie(cookie_value: &str) -> Result, String> .map_err(|e| format!("base64 decode failed: {e}"))?; if let Ok(eids) = serde_json::from_slice::>(&bytes) { - return Ok(legacy_cookie_eids_to_openrtb(eids)); + return Ok(DecodedCookieEids::Legacy(eids)); } let structured = serde_json::from_slice::>(&bytes) .map_err(|e| format!("JSON parse failed: {e}"))?; - Ok(structured_cookie_eids_to_openrtb(structured)) + Ok(DecodedCookieEids::Structured(structured)) +} + +impl DecodedCookieEids { + fn diagnostic_sources(&self) -> Vec { + match self { + Self::Legacy(entries) => entries + .iter() + .filter(|entry| !entry.source.is_empty()) + .map(|entry| DiagnosticEidSource { + source: entry.source.clone(), + uids: vec![entry.id.clone()], + }) + .collect(), + Self::Structured(entries) => entries + .iter() + .filter(|entry| !entry.source.is_empty()) + .map(|entry| DiagnosticEidSource { + source: entry.source.clone(), + uids: entry.uids.iter().map(|uid| uid.id.clone()).collect(), + }) + .collect(), + } + } + + fn into_openrtb(self) -> Vec { + match self { + Self::Legacy(entries) => legacy_cookie_eids_to_openrtb(entries), + Self::Structured(entries) => structured_cookie_eids_to_openrtb(entries), + } + } } /// Parses request-local EID cookies and writes matched partner UIDs to KV. @@ -179,13 +229,36 @@ pub(crate) fn collect_prebid_eid_updates( cookie_value: &str, registry: &PartnerRegistry, ) -> Vec { - let Ok(eids) = parse_prebid_eids_cookie(cookie_value) else { + let Ok(analysis) = analyze_prebid_eids_cookie(cookie_value, registry) else { log::trace!("Prebid EIDs: failed to decode ts-eids cookie; dropping"); return Vec::new(); }; + analysis.updates +} + +pub(crate) fn analyze_prebid_eids_cookie( + cookie_value: &str, + registry: &PartnerRegistry, +) -> Result { + let decoded = decode_prebid_eids_cookie(cookie_value)?; + let diagnostic_sources = decoded.diagnostic_sources(); + let eids = decoded.into_openrtb(); + let updates = collect_prebid_eid_updates_from_eids(&eids, registry); + + Ok(PrebidEidAnalysis { + eids, + diagnostic_sources, + updates, + }) +} + +fn collect_prebid_eid_updates_from_eids( + eids: &[Eid], + registry: &PartnerRegistry, +) -> Vec { let mut updates = Vec::new(); - for eid in &eids { + for eid in eids { let Some(partner) = registry.find_by_source_domain(&eid.source) else { log::debug!("Prebid EIDs: no partner for source '{}'", eid.source); continue; @@ -222,9 +295,11 @@ pub(crate) fn dedupe_partner_updates(updates: Vec) -> Vec Option<&Uid> { - uids.iter() - .filter(|uid| !uid.id.trim().is_empty()) - .find(|uid| !eid_id_exceeds_size_limit(&uid.id)) + uids.iter().find(|uid| is_valid_eid_uid(&uid.id)) +} + +pub(crate) fn is_valid_eid_uid(uid: &str) -> bool { + !uid.trim().is_empty() && !eid_id_exceeds_size_limit(uid) } /// `SharedID` EID source domain used for partner registry lookup. diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index de16aa758..2fe123fcb 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2293,18 +2293,19 @@ impl Settings { fn validate_admin_handler_passwords(&self) -> Result<(), Report> { for handler in &self.handlers { - let covers_admin = Self::ADMIN_ENDPOINTS - .iter() - .try_fold(false, |covers_any_endpoint, path| { - Self::admin_auth_probes(path).iter().try_fold( - covers_any_endpoint, - |covers_any_probe, probe| { - handler - .matches_path(probe) - .map(|matches| covers_any_probe || matches) - }, - ) - })?; + let covers_admin = + Self::ADMIN_ENDPOINTS + .iter() + .try_fold(false, |covers_any_endpoint, path| { + Self::admin_auth_probes(path).iter().try_fold( + covers_any_endpoint, + |covers_any_probe, probe| { + handler + .matches_path(probe) + .map(|matches| covers_any_probe || matches) + }, + ) + })?; if covers_admin && is_admin_placeholder_password(handler.password.expose()) { return Err(Report::new(TrustedServerError::Configuration { From 1198eb972771dbcab5a6e2c2af79164599708ef9 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 10:40:09 +0530 Subject: [PATCH 337/395] Share core request cookie extraction --- .../src/auction/endpoints.rs | 17 +------ crates/trusted-server-core/src/cookies.rs | 44 +++++++++++++++++++ crates/trusted-server-core/src/ec/admin.rs | 17 +------ 3 files changed, 46 insertions(+), 32 deletions(-) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index c4af6fd3d..326453157 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -11,6 +11,7 @@ use crate::auction::formats::AdRequest; use crate::auction::orchestrator::OrchestrationResult; use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::COOKIE_TS_EIDS; +use crate::cookies::extract_cookie_value; use crate::ec::EcContext; use crate::ec::eids::{resolve_partner_ids, to_eids}; use crate::ec::kv::KvIdentityGraph; @@ -408,22 +409,6 @@ pub(crate) fn resolve_auction_eids( Some(to_eids(&resolved)) } -fn extract_cookie_value(req: &Request, name: &str) -> Option { - let cookie_header = req - .headers() - .get(header::COOKIE) - .and_then(|v| v.to_str().ok())?; - for pair in cookie_header.split(';') { - let pair = pair.trim(); - if let Some((key, value)) = pair.split_once('=') - && key.trim() == name - { - return Some(value.trim().to_owned()); - } - } - None -} - pub(crate) fn resolve_client_auction_eids( raw: Option<&JsonValue>, cookie_value: Option<&str>, diff --git a/crates/trusted-server-core/src/cookies.rs b/crates/trusted-server-core/src/cookies.rs index 2ddc7a8b4..a716045f9 100644 --- a/crates/trusted-server-core/src/cookies.rs +++ b/crates/trusted-server-core/src/cookies.rs @@ -66,6 +66,19 @@ pub fn handle_request_cookies( } } +/// Returns the named value from the request's selected `Cookie` header. +/// +/// Values are trimmed and may contain additional `=` characters. A missing or +/// non-UTF-8 selected header returns `None`. +#[must_use] +pub fn extract_cookie_value(req: &Request, name: &str) -> Option { + let cookie_header = req.headers().get(header::COOKIE)?.to_str().ok()?; + cookie_header.split(';').find_map(|pair| { + let (key, value) = pair.trim().split_once('=')?; + (key.trim() == name).then(|| value.trim().to_owned()) + }) +} + /// Strips named cookies from a `Cookie` header value string. /// /// Parses the semicolon-separated cookie pairs, filters out any whose name @@ -242,6 +255,37 @@ mod tests { ); } + #[test] + fn extract_cookie_value_returns_none_without_cookie_header() { + let req = build_request(None); + + assert_eq!(extract_cookie_value(&req, "session"), None); + } + + #[test] + fn extract_cookie_value_trims_pairs_and_preserves_embedded_equals() { + let req = build_request(Some("first=one; token = abc== ; last=three")); + + assert_eq!( + extract_cookie_value(&req, "token").as_deref(), + Some("abc==") + ); + assert_eq!(extract_cookie_value(&req, "last").as_deref(), Some("three")); + } + + #[test] + fn extract_cookie_value_returns_none_for_selected_non_utf8_header() { + let invalid = HeaderValue::from_bytes(b"\xff=value").expect("should build header value"); + let mut req = build_request(None); + req.headers_mut().append(header::COOKIE, invalid); + req.headers_mut().append( + header::COOKIE, + HeaderValue::from_static("session=from-later-header"), + ); + + assert_eq!(extract_cookie_value(&req, "session"), None); + } + // --------------------------------------------------------------- // forward_cookie_header tests // --------------------------------------------------------------- diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index c5b5e2d59..9cc70fd04 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -26,6 +26,7 @@ use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt as _}; use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EC, COOKIE_TS_EIDS}; +use crate::cookies::extract_cookie_value; use crate::error::TrustedServerError; use crate::openrtb::Eid; @@ -546,22 +547,6 @@ pub fn handle_admin_eids_lookup( Ok(json_response(StatusCode::OK, body)) } -fn extract_cookie_value(req: &Request, name: &str) -> Option { - let cookie_header = req - .headers() - .get(header::COOKIE) - .and_then(|value| value.to_str().ok())?; - for pair in cookie_header.split(';') { - let pair = pair.trim(); - if let Some((key, value)) = pair.split_once('=') - && key.trim() == name - { - return Some(value.trim().to_owned()); - } - } - None -} - fn json_error(status: StatusCode, message: &str) -> Response { let body = serde_json::json!({ "error": message }); json_response(status, body.to_string()) From a3a04795ac8f69c73be61478a6887fbeeada87d2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 10:43:03 +0530 Subject: [PATCH 338/395] Harden admin diagnostic responses --- .../tests/routes.rs | 23 +++++++++++++++++++ .../tests/routes.rs | 16 +++++++++++++ .../trusted-server-adapter-fastly/src/app.rs | 13 +++++++++++ .../tests/routes.rs | 16 +++++++++++++ crates/trusted-server-core/src/ec/admin.rs | 12 +++++++++- 5 files changed, 79 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index bb4204ff9..a0452e255 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -314,6 +314,29 @@ async fn authenticated_admin_ec_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn admin_ec_route_without_credentials_returns_401() { + let mut svc = make_service(); + let req = Request::builder() + .method("GET") + .uri("/_ts/admin/ec") + .body(AxumBody::empty()) + .expect("should build unauthenticated admin EC request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + + assert_eq!(resp.status().as_u16(), 401); + assert!( + resp.headers().contains_key("www-authenticate"), + "admin EC 401 should include the Basic authentication challenge" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn authenticated_admin_eids_route_returns_200() { // The EIDs echo is pure request inspection (no KV), so the dev server diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index c512e2e9c..528d9348e 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -316,6 +316,22 @@ async fn authenticated_admin_ec_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn admin_ec_route_without_credentials_returns_401() { + let req = request_builder() + .method("GET") + .uri("/_ts/admin/ec") + .body(edgezero_core::body::Body::empty()) + .expect("should build unauthenticated admin EC request"); + let resp = route(test_router(), req).await; + + assert_eq!(resp.status().as_u16(), 401); + assert!( + resp.headers().contains_key("www-authenticate"), + "admin EC 401 should include the Basic authentication challenge" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn authenticated_admin_eids_route_returns_200() { // The EIDs echo is pure request inspection (no KV), so this adapter diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index ca0f528a2..c3be31f4c 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -2280,6 +2280,19 @@ mod tests { ); } + #[test] + fn admin_ec_route_without_credentials_returns_401() { + let router = test_router(); + + let response = route(&router, empty_request(Method::GET, "/_ts/admin/ec")); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert!( + response.headers().contains_key(header::WWW_AUTHENTICATE), + "admin EC 401 should include the Basic authentication challenge" + ); + } + #[test] fn dispatch_head_on_named_get_route_falls_through_to_publisher_fallback() { // Regression guard: HEAD /first-party/proxy must reach the publisher diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 68ac4d55a..e6737b6ba 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -151,6 +151,22 @@ async fn authenticated_admin_ec_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn admin_ec_route_without_credentials_returns_401() { + let req = request_builder() + .method("GET") + .uri("/_ts/admin/ec") + .body(edgezero_core::body::Body::empty()) + .expect("should build unauthenticated admin EC request"); + let resp = route(test_router(), req).await; + + assert_eq!(resp.status().as_u16(), 401); + assert!( + resp.headers().contains_key("www-authenticate"), + "admin EC 401 should include the Basic authentication challenge" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn authenticated_admin_eids_route_returns_200() { // The EIDs echo is pure request inspection (no KV), so this adapter diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 9cc70fd04..96fb708b6 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -113,7 +113,8 @@ struct AdminEcLookupResponse { /// Store generation marker for the entry. generation: u64, /// `true` when the entry is a consent-withdrawal tombstone - /// (`consent.ok = false`). Absent when the body failed to parse. + /// (`consent.ok = false`). Absent when the body failed to parse as JSON or + /// deserialize as a [`KvEntry`]. #[serde(skip_serializing_if = "Option::is_none")] tombstone: Option, /// The stored entry, preserved as raw JSON except for derived @@ -557,6 +558,7 @@ fn json_response(status: StatusCode, body: String) -> Response { .status(status) .header(header::CONTENT_TYPE, mime::APPLICATION_JSON.as_ref()) .header(header::CACHE_CONTROL, "no-store") + .header(header::X_CONTENT_TYPE_OPTIONS, "nosniff") .body(EdgeBody::from(body.into_bytes())) .expect("should build admin EC lookup response") } @@ -1070,6 +1072,10 @@ mod tests { response.headers().get(header::CACHE_CONTROL), Some(&HeaderValue::from_static("no-store")) ); + assert_eq!( + response.headers().get(header::X_CONTENT_TYPE_OPTIONS), + Some(&HeaderValue::from_static("nosniff")) + ); assert!( response_json(response)["error"] .as_str() @@ -1099,6 +1105,10 @@ mod tests { handle_admin_eids_lookup(&test_registry(), &req).expect("should handle eids lookup"); assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers().get(header::X_CONTENT_TYPE_OPTIONS), + Some(&HeaderValue::from_static("nosniff")) + ); let json = response_json(response); assert_eq!(json["cookie_present"], false); assert_eq!(json["sharedid_present"], false); From be434c2910ed54fde6a0a6b3fa62bad284bf2ab8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 10:43:58 +0530 Subject: [PATCH 339/395] Clarify admin diagnostics contracts --- CHANGELOG.md | 1 + docs/guide/api-reference.md | 23 +++++++++++++++++-- ...9-pr928-comprehensive-review-resolution.md | 2 +- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c00487769..4283dcbc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Breaking** — Admin Basic-auth coverage now includes `GET /_ts/admin/ec`, `GET /_ts/admin/ec/{id}`, and `GET /_ts/admin/eids`. Existing configurations whose `[[handlers]]` patterns protect only the key-management endpoints now fail startup; broaden coverage before deploying, preferably with a namespace-boundary pattern such as `^/_ts/admin(?:/|$)`. - **Breaking** — Replaced the legacy APS contextual integration with APS OpenRTB at `/e/pb/bid`. APS configuration now uses canonical `account_id` (`pub_id` remains a compatibility alias), no longer requires APS-specific slot IDs, and defaults script creative eligibility off. Operators must update the endpoint, disable native APS demand for Trusted Server cohorts, and prepare GAM/Universal Creative targeting for `hb_bidder=aps` before rollout. `aps` entries in Prebid bidder lists are logged and stripped. APS renderer winners now preserve the upstream bid `id`, omit `crid` when APS omits it, and carry `ext.trusted_server.renderer` instead of `adm`; external `/auction` consumers must support this response shape. - **Breaking** — All auction paths now forward only a validated publisher-owned page URL as `site.page`, removing query and fragment data. APS OpenRTB omits `site.ref`; the existing Prebid Server path continues to forward the browser `Referer` as `site.ref`. Query-driven sites may lose contextual targeting and per-page reporting signals that previously came from query parameters. - **Breaking** — `bid_param_zone_overrides` inner values must now be JSON objects; previously non-object or empty values (`"header" = "x"`, `"header" = {}`) were accepted and silently produced a dead rule at runtime. They now fail at startup with a configuration error. Operators upgrading should audit their `bid_param_zone_overrides` config for non-object zone entries. diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 2a30a1ac4..7c1801fc5 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -583,7 +583,7 @@ curl -X POST https://edge.example.com/_ts/admin/keys/deactivate \ ## Admin Diagnostic Endpoints -These endpoints expose sensitive identity and cookie data and require HTTP Basic Authentication. Configure a handler that covers the entire `/_ts/admin` namespace; startup rejects configurations that do not protect every admin route. All responses are JSON with `Cache-Control: no-store`. +These endpoints expose sensitive identity and cookie data and require HTTP Basic Authentication. Configure a handler that covers the entire `/_ts/admin` namespace; startup rejects configurations that do not protect every admin route. After successful authentication, diagnostic-handler responses are JSON with `Cache-Control: no-store`. Missing or invalid credentials receive the shared plaintext `401 Unauthorized` Basic-auth challenge, which is outside that JSON and cache-header contract. The examples below use fictional IDs and values only. @@ -630,7 +630,26 @@ curl -u admin:secure-password \ Parses the request's `ts-eids` and `sharedId` cookies and previews which configured partner IDs cookie ingestion would match or drop. It performs request inspection only: it does not read or write KV and is available on every adapter. -After successful authentication this endpoint always returns `200 OK`; missing or malformed cookies are represented by `cookie_present`, `sharedid_present`, and `parse_error`. The `ingest.matched` and `ingest.unmatched` arrays show the ingestion preview. +After successful authentication this endpoint always returns `200 OK`; missing or malformed cookies are represented by `cookie_present`, `sharedid_present`, and `parse_error`. The `ingest.matched` and `ingest.unmatched` arrays show the ingestion preview. Each unmatched entry contains its `source` and either a `no_partner` reason when no configured partner recognizes it or `no_valid_uid` when the partner exists but every supplied UID is empty or exceeds the storage limit. + +```json +{ + "ingest": { + "matched": [ + { + "source_domain": "configured.example", + "uid": "fictional-uid" + } + ], + "unmatched": [ + { + "source": "unknown.example", + "reason": "no_partner" + } + ] + } +} +``` ```bash curl -u admin:secure-password \ diff --git a/docs/superpowers/plans/2026-08-19-pr928-comprehensive-review-resolution.md b/docs/superpowers/plans/2026-08-19-pr928-comprehensive-review-resolution.md index 9095e7853..6305cfd35 100644 --- a/docs/superpowers/plans/2026-08-19-pr928-comprehensive-review-resolution.md +++ b/docs/superpowers/plans/2026-08-19-pr928-comprehensive-review-resolution.md @@ -183,7 +183,7 @@ git commit -m "Validate mixed-case admin EC auth coverage" Update the existing unmatched assertion to expect: ```json -{"source":"unknown.example","reason":"no_partner"} +{ "source": "unknown.example", "reason": "no_partner" } ``` Add a test whose configured source has only whitespace/empty and over-limit From 436e25f2392fbf9661049528e2629adc126e9974 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 10:51:15 +0530 Subject: [PATCH 340/395] Tighten admin diagnostic review coverage --- crates/trusted-server-adapter-fastly/src/app.rs | 5 +++++ docs/guide/api-reference.md | 5 +++-- ...026-08-19-pr928-comprehensive-review-resolution-design.md | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index c3be31f4c..072208fb2 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -2267,6 +2267,11 @@ mod tests { let response = route(&router, request); + assert_eq!( + response.status(), + StatusCode::NOT_IMPLEMENTED, + "configured admin EC handler should run and report the unavailable test KV graph" + ); assert!( response .extensions() diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 7c1801fc5..0fff97f8f 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -583,7 +583,7 @@ curl -X POST https://edge.example.com/_ts/admin/keys/deactivate \ ## Admin Diagnostic Endpoints -These endpoints expose sensitive identity and cookie data and require HTTP Basic Authentication. Configure a handler that covers the entire `/_ts/admin` namespace; startup rejects configurations that do not protect every admin route. After successful authentication, diagnostic-handler responses are JSON with `Cache-Control: no-store`. Missing or invalid credentials receive the shared plaintext `401 Unauthorized` Basic-auth challenge, which is outside that JSON and cache-header contract. +These endpoints expose sensitive identity and cookie data and require HTTP Basic Authentication. Configure a handler that covers the entire `/_ts/admin` namespace; startup rejects configurations that do not protect every admin route. Normal diagnostic-handler responses after successful authentication are JSON with `Cache-Control: no-store`. Missing or invalid credentials receive the shared plaintext `401 Unauthorized` Basic-auth challenge. Unexpected configuration or KV failures use the adapter's shared plaintext `5xx` error response. Those authentication and internal-error responses are outside the diagnostic JSON and cache-header contract. The examples below use fictional IDs and values only. @@ -600,7 +600,7 @@ This lookup is implemented only by the Fastly adapter because the identity graph - `ec_id`, `store`, and `generation` identify the raw KV lookup. - `entry` preserves the stored JSON shape, including unknown and legacy fields. Derived `created_iso` and `consent.updated_iso` fields are added only when absent. - `metadata` preserves the stored metadata JSON shape. -- `tombstone` reports whether consent has been withdrawn. +- `tombstone` reports whether consent has been withdrawn. It is absent when the entry body cannot be parsed as JSON or deserialized as the typed EC schema. - `auction.eids` previews the partner EIDs the stored record can contribute; `auction.skipped` explains filtered IDs. - `entry_error`, `metadata_error`, and `raw_body` keep malformed or schema-incompatible records inspectable. @@ -616,6 +616,7 @@ The auction preview validates the stored record and partner configuration, but c | `404` | Record not found, or the bare route has no `ts-ec` cookie | | `405` | Method other than `GET` (`Allow: GET`) | | `501` | EC identity graph unavailable on this adapter or deployment | +| `5xx` | Unexpected configuration or KV failure (plaintext) | ```bash curl -u admin:secure-password \ diff --git a/docs/superpowers/specs/2026-08-19-pr928-comprehensive-review-resolution-design.md b/docs/superpowers/specs/2026-08-19-pr928-comprehensive-review-resolution-design.md index ee87ea9fd..7a1363e99 100644 --- a/docs/superpowers/specs/2026-08-19-pr928-comprehensive-review-resolution-design.md +++ b/docs/superpowers/specs/2026-08-19-pr928-comprehensive-review-resolution-design.md @@ -1,7 +1,7 @@ # PR #928 Comprehensive Review Resolution -**PR:** #928 -**Date:** 2026-08-19 +**PR:** #928 +**Date:** 2026-08-19 **Status:** Approved design ## Problem From b8568f3cbcaf28443a3f208aa4c06789434b61be Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 11:41:58 +0530 Subject: [PATCH 341/395] Resolve ad-template CLI review findings --- Cargo.toml | 2 +- .../src/ad_templates/compare.rs | 10 +- .../src/ad_templates/expected.rs | 17 +- crates/trusted-server-cli/src/app_config.rs | 25 ++ .../commands/audit/ad_template_collector.js | 92 +++--- .../src/commands/audit/ad_templates.rs | 82 +++-- .../src/commands/audit/browser.rs | 30 +- .../src/commands/audit/collector.rs | 68 +++- .../src/commands/audit/consent_stub.js | 17 +- .../audit/generate/browser_collector.rs | 48 +-- .../src/commands/audit/generate/collector.rs | 6 +- .../src/commands/audit/generate/crawl_plan.rs | 108 ++++++- .../src/commands/audit/generate/evidence.rs | 22 +- .../src/commands/audit/generate/gpt_slots.rs | 111 +++++-- .../src/commands/audit/generate/mod.rs | 305 +++++++++++++++--- .../src/commands/audit/generate/slot_toml.rs | 185 ++++++----- .../commands/audit/generate/unit_template.rs | 16 +- .../src/commands/audit/mod.rs | 34 +- .../src/commands/config/ad_templates.rs | 67 ++-- crates/trusted-server-cli/src/run.rs | 64 +++- .../src/creative_opportunities.rs | 12 +- docs/guide/cli.md | 9 +- .../2026-08-18-pr-823-review-resolution.md | 2 +- ...6-26-server-side-ad-template-cli-design.md | 40 +-- scripts/test-cli.sh | 11 +- 25 files changed, 976 insertions(+), 407 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9647b562d..31b92923f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,10 +91,10 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus rustls = "0.23" rustls-pemfile = "2" scraper = "0.24.0" -similar = "2.7" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.149" sha2 = "0.10.9" +similar = "2.7" simple_logger = "5" spin-sdk = { version = "~6.0", default-features = false, features = ["http", "key-value", "variables"] } subtle = "2.6" diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs index eb5e66123..a1dc271a1 100644 --- a/crates/trusted-server-cli/src/ad_templates/compare.rs +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -305,7 +305,7 @@ pub fn compare_page_evidence( ), )); ( - SlotStatus::Unconfirmable, + SlotStatus::Partial, dom_id, Some(gpt.clone()), Some(gpt.phase), @@ -733,7 +733,7 @@ mod tests { } #[test] - fn out_of_page_gpt_slot_warns_and_does_not_confirm() { + fn sizeless_live_slot_is_partial_when_config_declares_banner_sizes() { let expected = expected_slot( "interstitial", "ad-oop-", @@ -753,7 +753,7 @@ mod tests { RuntimeGateSummary::unknown_allowed(), ); - assert_eq!(result.slots[0].status, SlotStatus::Unconfirmable); + assert_eq!(result.slots[0].status, SlotStatus::Partial); assert!( result.slots[0] .warnings @@ -761,8 +761,8 @@ mod tests { .any(|w| w.code == "out_of_page_slot") ); assert!( - !result.strict_failed(), - "out-of-page slots are not confirmable by this checker" + result.strict_failed(), + "a live sizeless slot drifting from configured banner sizes must fail strict" ); } diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs index 373283cb6..9392963ff 100644 --- a/crates/trusted-server-cli/src/ad_templates/expected.rs +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -136,7 +136,7 @@ pub fn normalize_path_or_url(input: &str) -> Result { .expect("should parse static path normalization base"); let relative = input.trim_start_matches('/'); let normalized = base - .join(relative) + .join(&format!("./{relative}")) .map_err(|error| format!("invalid path `{input}`: {error}"))?; Ok(normalized.path().to_string()) } @@ -256,8 +256,8 @@ mod tests { #[test] fn expected_slots_omit_dynamic_template_the_runtime_cannot_render() { // A `{section}` template that renders past GAM's 100-byte unit-path - // limit. `validate_runtime` rejects this config, so the verifier reports - // the slot as unconfirmable rather than matching a truncated path. + // limit. The runtime omits this slot for the request path, so diagnostics + // must not match it against a truncated or otherwise different path. let toml = "gam_network_id = \"99999\"\n\ section_root = \"homepage\"\n\ \n\ @@ -321,5 +321,16 @@ mod tests { .expect("query URL should not change input classification"), "/r" ); + assert_eq!( + normalize_path_or_url("/news:latest").expect("colon should stay in bare path"), + "/news:latest", + "a colon in the first segment must not be parsed as a URL scheme" + ); + assert_eq!( + normalize_path_or_url("https://example.com/news:latest") + .expect("colon should stay in URL path"), + "/news:latest", + "bare and absolute forms should normalize identically" + ); } } diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs index fadf58cde..a54cdfc72 100644 --- a/crates/trusted-server-cli/src/app_config.rs +++ b/crates/trusted-server-cli/src/app_config.rs @@ -62,10 +62,35 @@ pub fn load_settings(args: &AppConfigArgs) -> Result { /// /// Returns the same path-resolution, read, and parse errors as /// [`load_settings`]. +#[cfg(test)] pub fn load_file_settings(args: &AppConfigArgs) -> Result { load_settings_with_env_overlay(args, false) } +/// Resolves the operator-owned app-config path without deserializing settings. +/// +/// Mutating recovery commands use this when the existing config may already be +/// invalid but still needs a narrowly scoped structural repair. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded or has no +/// `[app].name` and no explicit config path was supplied. +pub fn resolve_app_config_file(args: &AppConfigArgs) -> Result { + if let Some(path) = &args.app_config { + return Ok(path.clone()); + } + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + Ok(resolve_app_config_path(None, &args.manifest, &app_name)) +} + fn load_settings_with_env_overlay( args: &AppConfigArgs, env_overlay: bool, diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js index ee610595d..46d1485d4 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -13,17 +13,16 @@ const __ts_ev = (window.__tsAdTemplateEvidence = window.__tsAdTemplateEvidence | dom_ids: [], gpt_slots: [], aps_calls: [], - warnings: [], + warnings: [] }) const __ts_phase = () => (window.__tsScrollPhase ? "scroll" : "initial_load") // Hard cap per evidence list so a hostile page cannot grow the store without // bound; the page controls how many slots/elements/warnings it produces. -const __ts_max_entries = 1024 +const __ts_max_entries = 128 const __ts_max_string_length = 512 const __ts_wrapped_googletags = new WeakSet() -const __ts_wrapped_apstags = new WeakSet() function __ts_text(value) { return String(value).slice(0, __ts_max_string_length) @@ -47,6 +46,16 @@ function __ts_size_pair(width, height) { return [width, height] } +function __ts_warn_ignored_size(width, height) { + const numeric = Number.isInteger(width) && Number.isInteger(height) + const outOfRange = + numeric && (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) + __ts_push(__ts_ev.warnings, { + code: outOfRange ? "size_out_of_range" : "fluid_size_ignored", + message: outOfRange ? "GPT size outside u32 range ignored" : "non-numeric GPT size ignored" + }) +} + function __ts_normalize_sizes(sizes) { const out = [] if (!Array.isArray(sizes)) return out @@ -58,10 +67,10 @@ function __ts_normalize_sizes(sizes) { if (pair) { out.push(pair) } else { - __ts_push(__ts_ev.warnings, { - code: "fluid_size_ignored", - message: "non-numeric GPT size ignored", - }) + __ts_warn_ignored_size( + Array.isArray(size) ? size[0] : undefined, + Array.isArray(size) ? size[1] : undefined + ) } } return out @@ -72,7 +81,7 @@ function __ts_record_define_slot(adUnitPath, sizes, divId) { gam_unit_path: __ts_text(adUnitPath), div_id: __ts_text(divId), sizes: __ts_normalize_sizes(sizes), - phase: __ts_phase(), + phase: __ts_phase() }) } @@ -87,9 +96,10 @@ function __ts_wrap_googletag(googletag) { const originalDefineSlot = googletag.defineSlot if (typeof originalDefineSlot === "function") { try { + const descriptor = Object.getOwnPropertyDescriptor(googletag, "defineSlot") Object.defineProperty(googletag, "defineSlot", { configurable: true, - enumerable: false, + enumerable: descriptor ? descriptor.enumerable : true, writable: true, value: function (adUnitPath, sizes, divId) { const slot = originalDefineSlot.apply(this, arguments) @@ -99,7 +109,7 @@ function __ts_wrap_googletag(googletag) { __ts_warn("define_slot_capture_failed", error) } return slot - }, + } }) } catch (error) { __ts_warn("define_slot_wrap_failed", error) @@ -108,40 +118,6 @@ function __ts_wrap_googletag(googletag) { return googletag } -function __ts_wrap_apstag(apstag) { - if (!apstag || (typeof apstag !== "object" && typeof apstag !== "function")) return apstag - if (__ts_wrapped_apstags.has(apstag)) return apstag - __ts_wrapped_apstags.add(apstag) - const originalFetchBids = apstag.fetchBids - if (typeof originalFetchBids === "function") { - try { - Object.defineProperty(apstag, "fetchBids", { - configurable: true, - enumerable: false, - writable: true, - value: function (config, callback) { - try { - const slots = (config && config.slots) || [] - for (const slot of slots) { - __ts_push(__ts_ev.aps_calls, { - slot_id: __ts_text(slot.slotID || slot.slotName || ""), - sizes: __ts_normalize_sizes(slot.sizes), - phase: __ts_phase(), - }) - } - } catch (error) { - __ts_warn("aps_capture_failed", error) - } - return originalFetchBids.apply(this, arguments) - }, - }) - } catch (error) { - __ts_warn("aps_wrap_failed", error) - } - } - return apstag -} - // Wrap an existing global or intercept a later assignment of it. function __ts_install(name, wrap) { if (window[name]) { @@ -165,12 +141,11 @@ function __ts_install(name, wrap) { } catch (error) { __ts_warn(name + "_wrap_failed", error) } - }, + } }) } __ts_install("googletag", __ts_wrap_googletag) -__ts_install("apstag", __ts_wrap_apstag) // On-demand DOM + getSlots scrape, invoked by the collector after settle/scroll. window.__tsCollectAdTemplateEvidence = function () { @@ -197,7 +172,11 @@ window.__tsCollectAdTemplateEvidence = function () { for (const size of rawSizes) { if (sizes.length >= __ts_max_entries) break let pair = null - if (size && typeof size.getWidth === "function" && typeof size.getHeight === "function") { + if ( + size && + typeof size.getWidth === "function" && + typeof size.getHeight === "function" + ) { // A fluid GPT size answers getWidth()/getHeight() with a // non-numeric value rather than throwing. pair = __ts_size_pair(size.getWidth(), size.getHeight()) @@ -207,10 +186,19 @@ window.__tsCollectAdTemplateEvidence = function () { if (pair) { sizes.push(pair) } else { - __ts_push(__ts_ev.warnings, { - code: "fluid_size_ignored", - message: "non-numeric GPT size ignored", - }) + const width = + size && typeof size.getWidth === "function" + ? size.getWidth() + : Array.isArray(size) + ? size[0] + : undefined + const height = + size && typeof size.getHeight === "function" + ? size.getHeight() + : Array.isArray(size) + ? size[1] + : undefined + __ts_warn_ignored_size(width, height) } } const exists = __ts_ev.gpt_slots.some( @@ -221,7 +209,7 @@ window.__tsCollectAdTemplateEvidence = function () { gam_unit_path: __ts_text(path), div_id: __ts_text(divId), sizes, - phase: __ts_phase(), + phase: __ts_phase() }) } } catch (error) { diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs index 7766db223..b36bd373f 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -37,6 +37,7 @@ use crate::run::RunOutcome; /// surfaces a page-level error or a `--strict` failure (after writing output). pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result { args.browser.validate()?; + validate_cookie_scope(&args.urls, &args.cookies)?; let loaded = crate::app_config::load_settings(&args.config)?; let collector = crate::commands::audit::browser::BrowserCollector::from_opts(&args.browser); let report = build_report( @@ -50,7 +51,7 @@ pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result Result Result<(), String> { + if cookies.is_empty() { + return Ok(()); + } + let origins: std::collections::BTreeSet = urls + .iter() + .map(|url| url.origin().ascii_serialization()) + .collect(); + if origins.len() > 1 { + return Err( + "--cookie may be used only when every verification URL has one origin; split this run so credentials are never copied to another origin" + .to_string(), + ); + } + Ok(()) +} + /// Run-level verification switches. #[derive(Debug, Clone, Copy)] struct VerifyOptions { @@ -91,14 +109,14 @@ fn build_report( urls: &[url::Url], options: VerifyOptions, cookies: &[(String, String)], -) -> VerificationReport { - let init_script = build_init_script(creative); +) -> Result { + let init_script = build_init_script(creative)?; let requests: Vec<_> = urls .iter() .map(|url| BrowserCollectRequest { url: url.clone(), - init_scripts: init_script.clone().into_iter().collect(), + init_scripts: vec![init_script.clone()], scroll: options.scroll, collect_ad_evidence: true, cookies: cookies.to_vec(), @@ -138,12 +156,12 @@ fn build_report( } let ok = !(any_error || (options.strict && any_strict_fail)); - VerificationReport { + Ok(VerificationReport { ok, strict: options.strict, pages, warnings: Vec::new(), - } + }) } /// Whether navigation left the requested URL's origin (scheme, host, or port). @@ -168,7 +186,7 @@ fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { } /// Builds the read-only collector init script from the configured slots. -fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Option { +fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Result { let config = AdTemplateCollectorConfig { div_prefixes: creative .map(|creative| { @@ -179,17 +197,8 @@ fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Option = collected.warnings.to_vec(); - warnings.extend(evidence.warnings.iter().cloned()); - if requested_path != final_path { + warnings.extend(evidence.warnings.iter().map(|warning| Warning { + code: format!("page_{}", warning.code), + message: warning.message.clone(), + })); + if requested != final_url { warnings.push(Warning { code: "redirected".to_string(), - message: format!("navigation redirected from {requested_path} to {final_path}"), + message: format!("navigation redirected from {requested} to {final_url}"), }); } @@ -669,6 +681,7 @@ mod tests { options, &[], ) + .expect("typed collector configuration should serialize") } #[test] @@ -842,7 +855,7 @@ mod tests { report.pages[0] .warnings .iter() - .any(|warning| warning.code == "fluid_size_ignored"), + .any(|warning| warning.code == "page_fluid_size_ignored"), "collector warning should be visible in the page report" ); } @@ -951,4 +964,31 @@ mod tests { assert_eq!(json["pages"][1]["error"]["code"], "navigation_failed"); assert!(json["pages"][1]["final_url"].is_null()); } + + #[test] + fn supplied_cookies_are_rejected_for_multiple_origins() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://b.example/y").expect("should parse second URL"), + ]; + + let error = validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect_err("should not replicate one cookie across origins"); + + assert!( + error.contains("one origin"), + "the refusal should explain cookie scope, got {error}" + ); + } + + #[test] + fn supplied_cookies_are_allowed_for_same_origin_urls() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://a.example/y").expect("should parse second URL"), + ]; + + validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect("same-origin URLs share the intended cookie scope"); + } } diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs index 1c8c37bf5..102443dce 100644 --- a/crates/trusted-server-cli/src/commands/audit/browser.rs +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -240,12 +240,12 @@ pub(crate) fn resolve_chrome( /// Builds a host-only cookie that applies to every path on `url`'s host. pub(crate) fn host_cookie(name: &str, value: &str, url: &url::Url) -> Result { - let host = url - .host_str() + url.host_str() .ok_or_else(|| format!("cannot scope cookie `{name}` because {} has no host", url))?; let mut cookie = CookieParam::new(name.to_string(), value.to_string()); - cookie.domain = Some(host.to_string()); + cookie.url = Some(url.to_string()); cookie.path = Some("/".to_string()); + cookie.secure = Some(url.scheme() == "https"); Ok(cookie) } @@ -382,11 +382,6 @@ impl AuditCollector for BrowserCollector { max: self.settle_max, }; - // chromiumoxide logs unparseable CDP events at WARN (its bundled CDP schema - // lags newer Chrome). These are benign; quiet them for the browser session - // so audit output stays clean, then restore the prior threshold. - let previous_level = log::max_level(); - log::set_max_level(log::LevelFilter::Error); let accept_invalid_certs = self.accept_invalid_certs; let headful = self.headful; let assume_consent = self.assume_consent; @@ -407,7 +402,6 @@ impl AuditCollector for BrowserCollector { }; collect(requests, &options).await }); - log::set_max_level(previous_level); match result { Ok(results) => results, Err(error) => vec![Err(error); request_count], @@ -511,6 +505,10 @@ async fn collect_open_page( page.evaluate_on_new_document(CONSENT_STUB_SCRIPT) .await .map_err(|error| format!("failed to install consent init script: {error}"))?; + warnings.push(Warning { + code: "consent_stub_active".to_string(), + message: "audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution".to_string(), + }); } page.evaluate_on_new_document("performance.setResourceTimingBufferSize(100000)") .await @@ -904,7 +902,7 @@ mod tests { /// Skips optional local runs, but makes the scripted/CI contract fail loudly. fn browser_fixture_available() -> bool { - if find_chrome().is_ok() { + if resolve_chrome(None).is_ok() { return true; } assert!( @@ -939,12 +937,14 @@ mod tests { url::Url::parse("https://publisher.example/news/story").expect("should parse test URL"); let cookie = host_cookie("clearance", "token", &url).expect("should build cookie"); - assert_eq!(cookie.domain.as_deref(), Some("publisher.example")); + assert!(cookie.domain.is_none(), "host-only cookies omit Domain"); assert_eq!(cookie.path.as_deref(), Some("/")); - assert!( - cookie.url.is_none(), - "domain/path and URL must not be combined" + assert_eq!( + cookie.url.as_deref(), + Some("https://publisher.example/news/story"), + "the URL scopes a host-only cookie before first navigation" ); + assert_eq!(cookie.secure, Some(true), "HTTPS cookies must be Secure"); } #[test] @@ -1092,7 +1092,6 @@ mod tests { let script = build_ad_template_init_script(&AdTemplateCollectorConfig { div_prefixes: vec!["ad-atf-".to_string()], - aps_slot_ids: Vec::new(), }) .expect("should build init script"); @@ -1139,7 +1138,6 @@ mod tests { let script = build_ad_template_init_script(&AdTemplateCollectorConfig { div_prefixes: vec!["ad-atf-".to_string()], - aps_slot_ids: Vec::new(), }) .expect("should build init script"); diff --git a/crates/trusted-server-cli/src/commands/audit/collector.rs b/crates/trusted-server-cli/src/commands/audit/collector.rs index e0ea26343..d803682ab 100644 --- a/crates/trusted-server-cli/src/commands/audit/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/collector.rs @@ -51,6 +51,40 @@ pub struct BrowserOpts { pub danger_accept_invalid_certs: bool, } +/// Browser options for generation, whose device selection is controlled by +/// `--profiles` rather than the verifier's singular `--browser-profile`. +#[derive(Debug, Clone, Args)] +pub struct GenerateBrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then auto-detection. + #[arg(long)] + pub chrome: Option, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds that marks the page settled. + #[arg(long, default_value_t = 750)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = 10_000)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +impl GenerateBrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } +} + /// Browser device profile shared by page audits and ad-template verification. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] pub enum BrowserProfile { @@ -64,14 +98,17 @@ pub enum BrowserProfile { impl BrowserOpts { /// Validates relationships between independently parsed browser flags. pub fn validate(&self) -> Result<(), String> { - if self.settle_quiet_ms > self.settle_max_ms { - return Err(format!( - "--settle-quiet-ms ({}) cannot exceed --settle-max-ms ({})", - self.settle_quiet_ms, self.settle_max_ms - )); - } - Ok(()) + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } +} + +fn validate_settle_window(quiet_ms: u64, max_ms: u64) -> Result<(), String> { + if quiet_ms > max_ms { + return Err(format!( + "--settle-quiet-ms ({quiet_ms}) cannot exceed --settle-max-ms ({max_ms})" + )); } + Ok(()) } /// A request to collect a single page. @@ -140,15 +177,12 @@ pub trait AuditCollector { /// Configuration handed to the read-only ad-template collector script. /// -/// Only the configured div prefixes and APS slot IDs are embedded — no page data -/// is requested. Serialized into the injected `__TS_CONFIG`. +/// Only configured div prefixes are embedded — no page data is requested. // Assembled by the ad-template verifier from the configured slots. #[derive(Debug, Clone, Default, serde::Serialize)] pub struct AdTemplateCollectorConfig { /// Configured slot div ID prefixes to match in the DOM. pub div_prefixes: Vec, - /// Configured APS slot IDs (reserved for provider-scoped filtering). - pub aps_slot_ids: Vec, } /// Builds the read-only ad-template init script, embedding `config` as `__TS_CONFIG`. @@ -177,7 +211,6 @@ mod tests { fn init_script_embeds_config_and_read_only_hooks() { let config = AdTemplateCollectorConfig { div_prefixes: vec!["ad-atf-".to_string()], - aps_slot_ids: vec!["atf".to_string()], }; let script = build_ad_template_init_script(&config).expect("should build script"); @@ -194,7 +227,7 @@ mod tests { !script.contains("ad-not-configured-"), "should not embed other prefixes" ); - // Bounded instrumentation markers (googletag/apstag wrapping + on-demand scrape). + // Bounded GPT instrumentation plus on-demand scrape. assert!( script.contains("__ts_install(\"googletag\""), "should install googletag hook" @@ -204,15 +237,16 @@ mod tests { "must not replace the publisher's variadic cmd.push" ); assert!(script.contains("defineSlot"), "should record defineSlot"); - assert!(script.contains("fetchBids"), "should wrap apstag.fetchBids"); + assert!(!script.contains("fetchBids"), "APS should not be mutated"); assert!( script.contains("new WeakSet()"), "should track wrapped objects without publisher-visible markers" ); assert!( - script.contains("Object.defineProperty(googletag, \"defineSlot\"") - && script.contains("enumerable: false"), - "wrapped methods should be non-enumerable" + script.contains("Object.getOwnPropertyDescriptor(") + && script.contains("\"defineSlot\"") + && script.contains("enumerable: descriptor ? descriptor.enumerable : true"), + "wrapped methods should preserve the publisher's enumerability" ); assert!( script.contains("4294967295"), diff --git a/crates/trusted-server-cli/src/commands/audit/consent_stub.js b/crates/trusted-server-cli/src/commands/audit/consent_stub.js index 05b0c9747..8a35da29e 100644 --- a/crates/trusted-server-cli/src/commands/audit/consent_stub.js +++ b/crates/trusted-server-cli/src/commands/audit/consent_stub.js @@ -1,4 +1,4 @@ -(() => { +;(() => { const tcData = { tcString: "", tcfPolicyVersion: 2, @@ -14,14 +14,14 @@ publisherCC: "US", purpose: { consents: {}, legitimateInterests: {} }, vendor: { consents: {}, legitimateInterests: {} }, - specialFeatureOptins: {}, + specialFeatureOptins: {} } for (let index = 1; index <= 10; index += 1) { tcData.purpose.consents[index] = true tcData.purpose.legitimateInterests[index] = true } - const tcfapi = (command, version, callback) => { + const tcfapi = (command, version, callback, _parameter) => { if (typeof callback !== "function") return switch (command) { case "ping": @@ -32,7 +32,7 @@ cmpStatus: "loaded", displayStatus: "hidden", apiVersion: "2.0", - cmpId: 0, + cmpId: 0 }, true ) @@ -57,9 +57,12 @@ const pin = (name, value) => { try { Object.defineProperty(window, name, { - value, - writable: false, - configurable: false, + get: () => value, + // Some CMP bundles assign these globals in strict mode. A no-op setter + // keeps the deterministic audit answer without throwing and aborting + // the publisher's CMP initialization. + set: () => {}, + configurable: false }) } catch (error) { // The page installed an earlier value; leave it untouched. diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index ebe02ab16..7a46a3e6c 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -15,7 +15,7 @@ use crate::commands::audit::browser::{ BrowserLaunchOptions, CONSENT_STUB_SCRIPT as SHARED_CONSENT_STUB_SCRIPT, build_browser_config, resolve_chrome, set_browser_cookies, }; -use crate::commands::audit::collector::BrowserOpts; +use crate::commands::audit::collector::GenerateBrowserOpts; use crate::commands::audit::generate::collector::{ AuditCollector, CollectedGptSlot, CollectedLink, CollectedPage, CollectedRequest, CollectedScriptTag, ControlFlow, PageSink, RootPlanner, @@ -33,9 +33,8 @@ const SETTLE_MAX_WAIT: Duration = Duration::from_secs(12); const NAVIGATION_LOAD_TIMEOUT: Duration = Duration::from_secs(12); const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); const PAGE_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); -const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; -const RESOURCE_TIMING_BUFFER_WARNING: &str = - "browser resource timing buffer reached its default size; some network assets may be missing"; +const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 100_000; +const RESOURCE_TIMING_BUFFER_WARNING: &str = "browser resource timing buffer reached its configured size; some network assets may be missing"; /// A device the crawl can emulate. /// @@ -154,7 +153,7 @@ impl Default for BrowserAuditCollector { impl BrowserAuditCollector { /// Applies the browser options shared by generate, verify, and page audit. #[must_use] - pub(crate) fn with_browser_options(mut self, options: &BrowserOpts) -> Self { + pub(crate) fn with_browser_options(mut self, options: &GenerateBrowserOpts) -> Self { self.chrome.clone_from(&options.chrome); self.headful = options.headful; self.assume_consent = !options.no_assume_consent; @@ -374,13 +373,7 @@ async fn with_browser( )) })?; - let handler_task = tokio::spawn(async move { - while let Some(event) = handler.next().await { - if event.is_err() { - break; - } - } - }); + let handler_task = tokio::spawn(async move { while handler.next().await.is_some() {} }); // Sitemap discovery is a whole-site fact, so only the first target pays for it. let mut result = Ok(()); @@ -433,9 +426,7 @@ async fn with_browser( report_error(format!("failed to close browser after audit: {error}")) }) }); - if close_result.is_err() { - handler_task.abort(); - } + handler_task.abort(); let _ = handler_task.await; match (result, close_result) { @@ -479,12 +470,19 @@ async fn collect_page_from_browser( match (result, close_result) { (Err(error), _) => Err(error), - (Ok(_), Err(_)) => Err(report_error( - "timed out closing browser tab after page collection", - )), - (Ok(_), Ok(Err(error))) => Err(report_error(format!( - "failed to close browser tab after page collection: {error}" - ))), + (Ok(mut collected), Err(_)) => { + collected.warnings.push( + "page_close_timeout: timed out closing browser tab after page collection" + .to_string(), + ); + Ok(collected) + } + (Ok(mut collected), Ok(Err(error))) => { + collected.warnings.push(format!( + "page_close_failed: failed to close browser tab after page collection: {error}" + )); + Ok(collected) + } (Ok(collected), Ok(Ok(_))) => Ok(collected), } } @@ -498,6 +496,8 @@ async fn collect_open_page( settle_quiet: Duration, settle_max: Duration, ) -> CliResult { + let mut warnings = Vec::new(); + // Must run before any page script, so the consent platform finds the APIs // already answered rather than installing its own gate. if assume_consent { @@ -506,6 +506,10 @@ async fn collect_open_page( .map_err(|error| { report_error(format!("failed to install the consent stub: {error}")) })?; + warnings.push( + "consent_stub_active: audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution" + .to_string(), + ); } page.evaluate_on_new_document("performance.setResourceTimingBufferSize(100000)") .await @@ -515,8 +519,6 @@ async fn collect_open_page( )) })?; - let mut warnings = Vec::new(); - // Navigate, but don't hard-fail when the `load` event never fires. Ad-heavy // pages (video players, continuous ad refresh, anti-bot scripts) can keep // the frame "loading" indefinitely, so a load-wait timeout is downgraded to diff --git a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs index 1ed9ce50e..7b9d87d0b 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs @@ -41,9 +41,9 @@ pub(crate) trait AuditCollector { /// crawl — a fresh launch per page dominates the cost of a multi-page run, /// and a shared profile carries bot-protection clearance cookies site-wide. /// - /// Results are streamed rather than returned as a `Vec` so the caller can - /// fold each page into its evidence and drop the page's HTML immediately, - /// instead of holding every DOM serialization at once. + /// Collectors may buffer results until the browser session closes so CPU-heavy + /// HTML analysis cannot starve a single-threaded CDP event pump. The sink API + /// keeps that buffering policy private and lets simple collectors stream. /// /// # Errors /// diff --git a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs index 6cb8c6c81..3f416278f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs @@ -49,7 +49,19 @@ const NOISE_SEGMENTS: &[&str] = &[ /// File extensions that are assets rather than pages. const NON_PAGE_EXTENSIONS: &[&str] = &[ ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".svg", ".ico", ".css", ".js", ".json", - ".xml", ".pdf", ".zip", ".mp4", ".mp3", ".rss", ".html", ".htm", ".php", + ".xml", ".pdf", ".zip", ".mp4", ".mp3", ".rss", +]; + +const DIRECTORY_INDEX_NAMES: &[&str] = &[ + "index.html", + "index.htm", + "index.php", + "default.html", + "default.htm", + "default.php", + "home.html", + "home.htm", + "home.php", ]; /// Bounds on how much of a site a single run will load. @@ -73,7 +85,7 @@ impl Default for CrawlBudget { /// One section selected for sampling. #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct PlannedSection { - /// The first path segment identifying the section (`news`). + /// The path segment at [`CrawlPlan::section_segment`] identifying the section. pub(super) segment: String, /// The section landing page, when one was observed. pub(super) landing: Option, @@ -148,12 +160,12 @@ pub(super) fn plan_crawl( sitemap_locs: &[String], budget: CrawlBudget, ) -> CrawlPlan { - let section_segment = usize::from(segment_count(root) == 1); + let section_segment = usize::from(root_is_locale_prefix(root)); let mut candidates: BTreeMap = BTreeMap::new(); let mut notes = Vec::new(); for link in links { - let Some(url) = same_origin_page_url(root, &link.url) else { + let Some(url) = same_origin_page_url(root, &link.url, section_segment) else { continue; }; let Some(segment) = section_at(&url, section_segment) else { @@ -167,7 +179,7 @@ pub(super) fn plan_crawl( let mut sitemap_pages = 0_usize; for loc in sitemap_locs { - let Some(url) = same_origin_page_url(root, loc) else { + let Some(url) = same_origin_page_url(root, loc, section_segment) else { continue; }; let Some(segment) = section_at(&url, section_segment) else { @@ -270,7 +282,7 @@ fn record_url(entry: &mut SectionCandidate, url: &Url, section_segment: usize) { /// Rejects other origins, non-HTTP schemes, asset extensions, and paginated or /// utility paths. Query and fragment are dropped so `/news?page=2` and /// `/news#top` collapse onto `/news`. -fn same_origin_page_url(root: &Url, raw: &str) -> Option { +fn same_origin_page_url(root: &Url, raw: &str, section_segment: usize) -> Option { let mut url = root.join(raw).ok()?; if !matches!(url.scheme(), "http" | "https") || url.origin() != root.origin() { return None; @@ -289,9 +301,22 @@ fn same_origin_page_url(root: &Url, raw: &str) -> Option { if segments.is_empty() { return None; } - if NOISE_SEGMENTS.contains(&segments[0]) { + if DIRECTORY_INDEX_NAMES.contains(&segments.last().copied().unwrap_or_default()) { + return None; + } + if NOISE_SEGMENTS.contains(&segments.get(section_segment).copied().unwrap_or_default()) { return None; } + if section_segment > 0 { + let root_path = percent_decode_for_filtering(root.path()).to_ascii_lowercase(); + let root_segments: Vec<&str> = root_path + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + if !segments.starts_with(&root_segments) { + return None; + } + } // `/news/page/2` is the same inventory as `/news`, so it is not a second // sample worth spending a page load on. if segments.contains(&"page") { @@ -300,15 +325,35 @@ fn same_origin_page_url(root: &Url, raw: &str) -> Option { Some(url) } -/// The first non-empty path segment, lowercased. +/// The non-empty path segment at `index`, percent-decoded and lowercased. fn section_at(url: &Url, index: usize) -> Option { - url.path() + percent_decode_for_filtering(url.path()) .split('/') .filter(|part| !part.is_empty()) .nth(index) .map(str::to_ascii_lowercase) } +fn root_is_locale_prefix(root: &Url) -> bool { + let segments: Vec<&str> = root + .path() + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + matches!(segments.as_slice(), [locale] if is_locale_segment(locale)) +} + +fn is_locale_segment(segment: &str) -> bool { + let bytes = segment.as_bytes(); + matches!(bytes, [a, b] if a.is_ascii_alphabetic() && b.is_ascii_alphabetic()) + || matches!(bytes, [a, b, b'-', c, d] + if a.is_ascii_alphabetic() + && b.is_ascii_alphabetic() + && c.is_ascii_alphabetic() + && d.is_ascii_alphabetic()) +} + +/// Decodes percent escapes solely for normalized path classification. fn percent_decode_for_filtering(path: &str) -> String { let bytes = path.as_bytes(); let mut decoded = Vec::with_capacity(bytes.len()); @@ -329,6 +374,7 @@ fn percent_decode_for_filtering(path: &str) -> String { String::from_utf8_lossy(&decoded).into_owned() } +/// Converts one ASCII hexadecimal digit to its numeric value. fn hex_value(byte: u8) -> Option { match byte { b'0'..=b'9' => Some(byte - b'0'), @@ -593,7 +639,49 @@ mod tests { CrawlBudget::default(), ); - assert_eq!(segments(&plan), ["news"]); + assert_eq!( + segments(&plan), + ["archive.htm", "news", "story.php"], + "only directory-index documents should be excluded by extension" + ); + } + + #[test] + fn locale_root_rejects_candidates_outside_its_path_prefix() { + let locale_root = Url::parse("https://publisher.example/en").expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav("/en/news"), nav("/fr/deals")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "a locale-root crawl must not mix another locale on the same origin" + ); + } + + #[test] + fn a_section_root_does_not_treat_article_slugs_as_sections() { + let section_root = Url::parse("https://publisher.example/news").expect("should parse root"); + let plan = plan_crawl( + §ion_root, + &[nav("/news/story-one"), nav("/news/story-two")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 0, + "a generic one-segment root is a section, not necessarily a locale" + ); + assert_eq!( + segments(&plan), + ["news"], + "articles below a section root should remain one section" + ); } #[test] diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs index 62dea746a..43bb5e6fa 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -130,6 +130,8 @@ pub(super) struct EvidenceTable { pages: BTreeSet, /// Page paths that produced no slot evidence at all. empty_pages: BTreeSet, + /// Page paths that produced slot evidence on at least one selected profile. + non_empty_pages: BTreeSet, } impl EvidenceTable { @@ -144,9 +146,12 @@ impl EvidenceTable { self.network_ids.insert(network_id.clone()); } if discovered.slots.is_empty() { - self.empty_pages.insert(path.to_string()); + if !self.non_empty_pages.contains(path) { + self.empty_pages.insert(path.to_string()); + } return; } + self.non_empty_pages.insert(path.to_string()); self.empty_pages.remove(path); for slot in &discovered.slots { @@ -533,6 +538,21 @@ mod tests { assert!(table.empty_pages().is_empty()); } + #[test] + fn a_later_empty_profile_does_not_re_mark_a_non_empty_page() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/news", + &page(&[("/99/site/news", "ad-atf", &[(300, 250)])], false), + ); + table.fold_page("/news", &page(&[], false)); + + assert!( + table.empty_pages().is_empty(), + "emptiness is a page-level fact across all selected profiles" + ); + } + #[test] fn conflicting_network_ids_are_a_hard_error() { let mut table = EvidenceTable::default(); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index 23d27624a..af5eb4f2f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -87,6 +87,8 @@ pub(crate) struct DiscoveredSlots { pub(crate) gam_network_id: Option, /// The reconstructed slots, deduplicated by div id in first-seen order. pub(crate) slots: Vec, + /// Diagnostics for placements whose normalized stable stems collided. + pub(crate) warnings: Vec, } /// Reconstructs GPT slots from the page's live registry and ad requests. @@ -105,24 +107,39 @@ pub(crate) fn discover_gpt_slots( page_has_prebid: bool, ) -> DiscoveredSlots { let mut slots = Vec::new(); + let mut warnings = Vec::new(); let mut gam_network_id = None; - let mut seen_divs: BTreeMap = BTreeMap::new(); + let mut registry_divs: BTreeMap> = BTreeMap::new(); for entry in registry { let Some(slot) = slot_from_registry(entry, page_has_prebid) else { continue; }; - push_slot_preserving_collisions(&mut slots, &mut seen_divs, slot, &entry.div_id); + if push_slot_preserving_collisions(&mut slots, &mut registry_divs, slot, &entry.div_id) { + warnings.push(format!( + "normalized div-id collision retained raw volatile id `{}`; it may not match a later render", + entry.div_id + )); + } if gam_network_id.is_none() { gam_network_id = network_id_from_unit_path(&entry.gam_unit_path); } } + let registry_stems: BTreeSet = registry_divs.keys().cloned().collect(); + let mut request_divs: BTreeMap> = BTreeMap::new(); for request in requests { let Some((network_id, slot, raw_div)) = parse_gampad_request(&request.url) else { continue; }; - push_slot_preserving_collisions(&mut slots, &mut seen_divs, slot, &raw_div); + if registry_stems.contains(&slot.div_id) { + continue; + } + if push_slot_preserving_collisions(&mut slots, &mut request_divs, slot, &raw_div) { + warnings.push(format!( + "normalized request div-id collision retained raw volatile id `{raw_div}`; it may not match a later render" + )); + } if gam_network_id.is_none() { gam_network_id = Some(network_id); } @@ -132,31 +149,43 @@ pub(crate) fn discover_gpt_slots( DiscoveredSlots { gam_network_id, slots, + warnings, } } +/// Adds one source-local slot while retaining distinct raw div IDs that share a stem. +/// +/// Returns whether a normalized collision forced raw, potentially volatile IDs +/// to be retained for both placements. fn push_slot_preserving_collisions( slots: &mut Vec, - seen_divs: &mut BTreeMap, + seen_divs: &mut BTreeMap>, mut slot: DiscoveredSlot, raw_div: &str, -) { +) -> bool { let normalized = slot.div_id.clone(); let raw_div = raw_div.strip_suffix("-container").unwrap_or(raw_div); - match seen_divs.get(&normalized) { + match seen_divs.get_mut(&normalized) { None => { - seen_divs.insert(normalized, raw_div.to_string()); + seen_divs.insert(normalized, BTreeSet::from([raw_div.to_string()])); slots.push(slot); + false } - Some(previous_raw) if previous_raw == raw_div => {} - Some(previous_raw) => { + Some(raw_divs) if raw_divs.contains(raw_div) => false, + Some(raw_divs) => { + let previous_raw = raw_divs + .first() + .expect("should have a first raw div after initial insertion") + .clone(); if let Some(previous) = slots.iter_mut().find(|entry| entry.div_id == normalized) { - previous.div_id.clone_from(previous_raw); - previous.id = slot_id_from_div(previous_raw); + previous.div_id.clone_from(&previous_raw); + previous.id = slot_id_from_div(&previous_raw); } slot.div_id = raw_div.to_string(); slot.id = slot_id_from_div(raw_div); + raw_divs.insert(raw_div.to_string()); slots.push(slot); + true } } } @@ -232,14 +261,14 @@ fn normalize_div_stem(div_id: &str) -> String { if let Some(matched) = REACT_USE_ID.find(stem) { cut = cut.min(matched.start()); } - if let Some(matched) = UUID_SEGMENT.find(stem).or_else(|| { - HEX_HASH_SEGMENT.find_iter(stem).find(|matched| { - matched - .as_str() - .bytes() - .any(|byte| matches!(byte, b'a'..=b'f')) - }) - }) { + let uuid = UUID_SEGMENT.find(stem); + let hex = HEX_HASH_SEGMENT.find_iter(stem).find(|matched| { + matched + .as_str() + .bytes() + .any(|byte| matches!(byte, b'a'..=b'f')) + }); + if let Some(matched) = uuid.into_iter().chain(hex).min_by_key(regex::Match::start) { cut = cut.min(matched.start()); } stem[..cut].trim_end_matches('-').to_string() @@ -910,5 +939,49 @@ mod tests { ); assert_eq!(discovered.slots[0].formats, vec![(300, 250)]); assert_ne!(discovered.slots[0].id, discovered.slots[1].id); + assert_eq!( + discovered.warnings.len(), + 1, + "writing raw volatile IDs must be diagnosable" + ); + } + + #[test] + fn repeated_raw_div_after_a_normalization_collision_is_deduplicated() { + let first = "ad-x-aaaaaaaaaaaaaaaa-0"; + let second = "ad-x-bbbbbbbbbbbbbbbb-1"; + let registry = vec![ + registry_slot("/123456789/site/home", first, &[(300, 250)]), + registry_slot("/123456789/site/home", second, &[(300, 250)]), + registry_slot("/123456789/site/home", second, &[(300, 250)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 2, + "an exact raw div repeat must remain first-seen deduplicated" + ); + } + + #[test] + fn request_rerender_does_not_rewrite_a_stable_registry_slot() { + let registry = vec![registry_slot( + "/123456789/site/home", + "ad-x-aaaaaaaaaaaaaaaa-0", + &[(300, 250)], + )]; + let requests = vec![request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-bbbbbbbbbbbbbbbb-1&prev_iu_szs=300x250", + )]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + assert_eq!(discovered.slots.len(), 1, "registry evidence should win"); + assert_eq!( + discovered.slots[0].div_id, "ad-x", + "request fallback must not destabilize a registry-derived prefix" + ); } } diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 5af2c8a66..4b506939d 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -481,15 +481,6 @@ fn render_discovered_slots(target_url: &Url, slots: &gpt_slots::DiscoveredSlots) out } -/// Runs `ts audit ad-templates generate`: scrape the live page's GPT slots and -/// rewrite only the `[creative_opportunities]` slot array in `config_path` in -/// place, preserving every other section and comment. -/// -/// # Errors -/// -/// Returns an error when the config cannot be read, the page cannot be -/// collected, no slots are discovered, or the config has no -/// `[creative_opportunities]` section to update. /// Everything one `ts audit ad-templates generate` invocation needs. pub(crate) struct UpdateSlotsRequest<'a> { /// Page URL to start from; also bounds the crawl to its origin. @@ -557,6 +548,12 @@ pub(crate) fn run_update_slots( request.cookies, &mut |_, root| { root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); + if root_url.origin() != target_url.origin() { + return cli_error(format!( + "refusing cross-origin root redirect from {} to {}; the requested origin is the audit and cookie trust boundary", + target_url, root_url + )); + } let plan = crawl_plan::plan_crawl(&root_url, &root.links, &root.sitemap_locs, request.budget); let targets = plan.targets(); @@ -573,8 +570,7 @@ pub(crate) fn run_update_slots( } } Err(error) => { - fold_error = Some(error); - return Ok(collector::ControlFlow::Stop); + notes.push(format!("skipped `{url}` on {first_label}: {error}")); } } Ok(collector::ControlFlow::Continue) @@ -594,7 +590,7 @@ pub(crate) fn run_update_slots( // disagree about a slot's ad-unit path, that shows up as two observations of // one page, which inference already refuses to represent. for (label, collector) in collectors.iter().skip(1) { - crawl_sections( + let successful_pages = crawl_sections( *collector, &root_url, &plan, @@ -603,6 +599,11 @@ pub(crate) fn run_update_slots( &mut notes, label, )?; + if successful_pages == 0 { + return cli_error(format!( + "the selected {label} device profile did not collect any required page; refusing to generate from incomplete profile coverage" + )); + } } if collectors.len() > 1 { notes.push(format!( @@ -648,6 +649,7 @@ pub(crate) fn run_update_slots( let policy = inference .as_ref() .and_then(|outcome| outcome.policy.clone()); + validate_merge_policy(request.existing_creative, policy.as_ref(), request.replace)?; // Slots that are one placement wearing a per-render div id cannot be // written: the ids never match at runtime. Report them so the operator can @@ -673,6 +675,7 @@ pub(crate) fn run_update_slots( inference.as_ref(), policy.as_ref(), request, + plan.section_segment, &fragmented, &mut notes, )?; @@ -682,6 +685,12 @@ pub(crate) fn run_update_slots( request.replace, ); notes.extend(merge_diagnostics); + if merged.is_empty() { + emit_notes(err, &mut notes)?; + return cli_error( + "refusing to write zero generated slots after the crawl discovered slot evidence; review the refused-slot notes and keep the existing configuration", + ); + } let rendered_slots = render_slots(&merged); let updated = splice_creative_slots( &existing, @@ -715,6 +724,12 @@ pub(crate) fn run_update_slots( let old_managed = managed_creative_projection(&existing)?; let new_managed = managed_creative_projection(&updated)?; let diff = similar::TextDiff::from_lines(&old_managed, &new_managed); + if old_managed == new_managed { + writeln!(out, "No managed creative-opportunity changes.").map_err(|error| { + report_error(format!("failed to write preview output: {error}")) + })?; + return Ok(()); + } writeln!( out, "{}", @@ -812,8 +827,12 @@ fn looks_like_an_interstitial(artifact: &AuditArtifact) -> Option { /// Writes and clears the pending notes, so each is reported exactly once. fn emit_notes(out: &mut dyn Write, notes: &mut Vec) -> CliResult<()> { for note in notes.drain(..) { - writeln!(out, "note: {note}") - .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + writeln!( + out, + "note: {}", + crate::ad_templates::output::escape_terminal_text(¬e) + ) + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; } Ok(()) } @@ -849,6 +868,7 @@ fn fold_collected( &collected.network_requests, page_has_prebid, ); + notes.extend(discovered.warnings.iter().cloned()); table.fold_page(url.path(), &discovered); Ok(()) } @@ -867,7 +887,7 @@ fn crawl_sections( table: &mut evidence::EvidenceTable, notes: &mut Vec, profile_label: &str, -) -> CliResult<()> { +) -> CliResult { let additional_targets = plan.targets(); if additional_targets.is_empty() { notes.push( @@ -883,9 +903,11 @@ fn crawl_sections( targets.extend(additional_targets); let mut fold_error = None; + let mut successful_pages = 0_usize; collector.collect_pages(&targets, cookies, &mut |url, collected| { match collected { Ok(page) => { + successful_pages += 1; let final_url = page.final_url().unwrap_or_else(|_| url.clone()); if let Err(error) = fold_collected(table, &final_url, &page, notes) { fold_error = Some(error); @@ -898,7 +920,7 @@ fn crawl_sections( })?; match fold_error { Some(error) => Err(error), - None => Ok(()), + None => Ok(successful_pages), } } @@ -918,12 +940,43 @@ fn guard_challenge_rate(table: &evidence::EvidenceTable) -> CliResult<()> { )) } +fn validate_merge_policy( + existing: Option<&CreativeOpportunitiesConfig>, + inferred: Option<&unit_template::SectionPolicy>, + replace: bool, +) -> CliResult<()> { + if replace { + return Ok(()); + } + let Some(existing) = existing else { + return Ok(()); + }; + let preserves_template = existing.slot.iter().any(|slot| { + slot.gam_unit_path + .as_deref() + .is_some_and(|path| path.contains("{section}")) + }); + let Some(inferred) = inferred.filter(|_| preserves_template) else { + return Ok(()); + }; + let configured_root = existing.section_root.as_deref().unwrap_or_default(); + let configured_segment = existing.section_segment.unwrap_or(0); + if configured_root != inferred.section_root || configured_segment != inferred.section_segment { + return cli_error(format!( + "refusing to change the section policy used by preserved templated slots during merge: configured section_root={configured_root:?}, section_segment={configured_segment}; inferred section_root={:?}, section_segment={}. Re-run with --replace only for an intentional migration", + inferred.section_root, inferred.section_segment + )); + } + Ok(()) +} + /// Turns the evidence table into slots ready to render. fn build_render_slots( table: &evidence::EvidenceTable, inference: Option<&unit_template::InferenceOutcome>, policy: Option<&unit_template::SectionPolicy>, request: &UpdateSlotsRequest<'_>, + fallback_section_segment: usize, fragmented: &[evidence::FragmentGroup], notes: &mut Vec, ) -> CliResult> { @@ -937,7 +990,7 @@ fn build_render_slots( if explicit { validate_page_patterns(request.page_patterns)?; } - let section_segment = policy.map_or(0, |policy| policy.section_segment); + let section_segment = policy.map_or(fallback_section_segment, |policy| policy.section_segment); let mut slots = Vec::with_capacity(table.slot_count()); for slot in table.slots() { @@ -978,7 +1031,7 @@ fn build_render_slots( } /// Rejects any page pattern the runtime's glob compiler would not accept. /// -/// Uses [`compile_page_pattern`] so the accepted set is exactly what +/// Uses [`validate_page_pattern`] so the accepted set is exactly what /// `CreativeOpportunitySlot::compile_patterns` accepts at startup, including the /// `**`→`*` normalisation. All patterns are reported at once so an operator /// passing several `--page-pattern` values fixes them in one pass. @@ -1062,6 +1115,18 @@ mod tests { visited: std::cell::RefCell>, } + struct FailingCollector; + + impl AuditCollector for FailingCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + cli_error(format!("simulated navigation failure for {target_url}")) + } + } + impl SiteCollector { fn new(pages: Vec<(&str, CollectedPage)>) -> Self { Self { @@ -1181,6 +1246,28 @@ mod tests { } } + #[test] + fn merge_refuses_to_change_policy_used_by_preserved_templates() { + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\nsection_root = \"home\"\nsection_segment = 0\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let inferred = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + + let error = validate_merge_policy(Some(&existing), Some(&inferred), false) + .expect_err("merge must preserve the existing template policy"); + + assert!(format!("{error:?}").contains("--replace")); + validate_merge_policy(Some(&existing), Some(&inferred), true) + .expect("replace is an explicit policy migration"); + } + #[test] fn resolve_output_plan_rejects_no_outputs() { let mut args = audit_args("https://publisher.example"); @@ -1527,6 +1614,133 @@ mod tests { ); } + #[test] + fn static_locale_root_slot_uses_the_planned_section_depth_for_patterns() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/en/news"]; + let mut root_page = site_page("https://publisher.example/en", "/123456789/site/root", &nav); + root_page.gpt_slots[0].div_id = "ad-root-only".to_string(); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/en", root_page), + ( + "https://publisher.example/en/news", + site_page( + "https://publisher.example/en/news", + "/123456789/site/static", + &nav, + ), + ), + ]); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/en", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect("should write static locale-root slot"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse config"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("should have slots"); + let target = slots + .iter() + .find(|slot| slot["div_id"].as_str() == Some("ad-header-0")) + .expect("should have the section slot"); + let patterns = target["page_patterns"] + .as_array() + .expect("should have patterns") + .iter() + .map(|pattern| pattern.as_str().expect("should be string")) + .collect::>(); + assert_eq!(patterns, ["/en/news", "/en/news/*"]); + } + + #[test] + fn update_slots_rejects_a_cross_origin_root_redirect() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.final_url = "https://foreign.example/news".to_string(); + let collector = FakeCollector::new(collected); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("cross-origin redirect must leave the requested trust boundary"); + + assert!(format!("{error:?}").contains("cross-origin")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "foreign evidence must not rewrite the config" + ); + } + + #[test] + fn update_slots_requires_evidence_from_every_selected_profile() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + let desktop = FakeCollector::new(collected_page_with_header_slot()); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &FailingCollector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("a selected profile with no usable page must refuse generation"); + + assert!(format!("{error:?}").contains("mobile")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "incomplete profile coverage must not rewrite the config" + ); + } + #[test] fn update_slots_rejects_invalid_page_pattern_without_touching_config() { let temp = TempDir::new().expect("should create temp dir"); @@ -1675,7 +1889,8 @@ mod tests { // infer `{section}`, and write a config the runtime loads. let temp = TempDir::new().expect("should create temp dir"); let config_path = temp.path().join("trusted-server.toml"); - fs::write(&config_path, loadable_config()).expect("should write config"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); let nav = ["/news", "/deals"]; let collector = SiteCollector::new(vec![ @@ -1770,7 +1985,8 @@ mod tests { // would be correct for one device and silently wrong for the other. let temp = TempDir::new().expect("should create temp dir"); let config_path = temp.path().join("trusted-server.toml"); - fs::write(&config_path, loadable_config()).expect("should write config"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); let nav = ["/news"]; let desktop = SiteCollector::new(vec![ @@ -1812,7 +2028,7 @@ mod tests { let mut out = Vec::new(); let mut err = Vec::new(); - run_update_slots( + let error = run_update_slots( &UpdateSlotsRequest { url: "https://publisher.example/", config_path: &config_path, @@ -1827,29 +2043,18 @@ mod tests { &mut out, &mut err, ) - .expect("the run should complete and report the conflict"); + .expect_err("an all-refused crawl must not write an empty slot array"); - let written = fs::read_to_string(&config_path).expect("read config"); - let value = toml::from_str::(&written).expect("valid TOML"); - let creative = &value["creative_opportunities"]; - assert!( - creative.get("section_root").is_none(), - "a device split must not produce a section template" - ); - assert!( - creative - .get("slot") - .and_then(toml::Value::as_array) - .is_none_or(Vec::is_empty), - "a refused device-split slot must be omitted, got:\n{written}" - ); + assert!(format!("{error:?}").contains("zero generated slots")); assert!( String::from_utf8_lossy(&err).contains("skipped refused slot"), "the refusal reason should be reported" ); - // What was written must still load. - trusted_server_core::settings::Settings::from_toml(&written) - .expect("a config with the refused slot omitted should still load"); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a refused crawl must preserve the operator config" + ); } #[test] @@ -1860,7 +2065,8 @@ mod tests { // every device agreed with it. let temp = TempDir::new().expect("should create temp dir"); let config_path = temp.path().join("trusted-server.toml"); - fs::write(&config_path, loadable_config()).expect("should write config"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); let desktop = SiteCollector::new(vec![( "https://publisher.example/", @@ -1880,7 +2086,7 @@ mod tests { )]); let mut out = Vec::new(); - run_update_slots( + let error = run_update_slots( &UpdateSlotsRequest { url: "https://publisher.example/", config_path: &config_path, @@ -1895,24 +2101,19 @@ mod tests { &mut out, &mut std::io::sink(), ) - .expect("the run should complete and report the conflict"); + .expect_err("an all-refused crawl must not write an empty slot array"); assert_eq!( mobile.visited.borrow().as_slice(), ["https://publisher.example/"], "the mobile profile must load the root even when there is nothing else to crawl" ); - let written = fs::read_to_string(&config_path).expect("read config"); - let value = toml::from_str::(&written).expect("valid TOML"); - assert!( - value["creative_opportunities"] - .get("slot") - .and_then(toml::Value::as_array) - .is_none_or(Vec::is_empty), - "a root-only device split must omit the refused slot, got:\n{written}" + assert!(format!("{error:?}").contains("zero generated slots")); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a root-only refusal must preserve the operator config" ); - trusted_server_core::settings::Settings::from_toml(&written) - .expect("a config with the refused slot omitted should still load"); } #[test] diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 0e4844daa..e2d614f4f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -67,9 +67,9 @@ impl RenderSlot { /// Builds a slot from cross-page evidence and the inferred unit path. /// - /// `gam_unit_path` is `None` when inference refused to represent the slot; - /// the slot is still written so its div and formats are not lost, and the - /// runtime falls back to the default `//` path. + /// Refused inference decisions are filtered before this constructor. A + /// `None` path therefore means inference was unavailable and deliberately + /// leaves the runtime's configured default-path behavior in effect. pub(super) fn from_evidence( id: &str, div_id: &str, @@ -144,6 +144,8 @@ fn media_type_label(media_type: &MediaType) -> Option<&'static str> { /// - Otherwise existing slots are preserved (covering other pages / hand-tuned /// fields); a slot re-seen this run has its page patterns and formats unioned; /// slots seen only this run are appended. +/// - Format identity includes media type, so equal dimensions observed for two +/// media types remain two intentional entries. #[cfg(test)] pub(super) fn merge_slots( existing: Option<&CreativeOpportunitiesConfig>, @@ -161,7 +163,7 @@ pub(super) fn merge_slots( /// Merges already-built slots into the existing set. /// -/// Same reconciliation as [`merge_slots`], but the caller supplies the slots — +/// Same reconciliation as the single-page test helper, but the caller supplies the slots — /// the crawl path builds them from cross-page evidence rather than from one /// page's discoveries. A slot re-seen this run keeps its configured fields and /// gains this run's patterns; a genuinely new slot is appended with a @@ -285,9 +287,7 @@ fn matching_slot_index(existing: &[RenderSlot], discovered: &RenderSlot) -> Opti existing.iter().position(|slot| slot.key() == key) } -/// Header comment emitted above the managed slot array. Stripped from the -/// preserved scalar block on re-splice (see [`is_managed_comment_line`]) so -/// repeated `generate` runs don't accumulate duplicate copies. +/// Header comment emitted above the structurally replaced managed slot array. const MANAGED_SLOTS_COMMENT: &str = "# Slots managed by `ts audit ad-templates generate`."; /// Second line of the managed-slot header comment. const MANAGED_SLOTS_REVIEW_COMMENT: &str = @@ -541,7 +541,7 @@ pub(super) fn splice_creative_slots( let mut result = document.to_string(); if uses_crlf(existing) { - result = result.replace("\r\n", "\n").replace('\n', "\r\n"); + result = convert_document_lf_to_crlf(&result); } ensure_only_managed_fields_changed(existing, &result)?; Ok(result) @@ -582,7 +582,60 @@ fn ensure_only_managed_fields_changed(before: &str, after: &str) -> CliResult<() /// Whether `document` uses CRLF line endings (so edits preserve them). fn uses_crlf(document: &str) -> bool { - document.contains("\r\n") + let mut multiline: Option = None; + let bytes = document.as_bytes(); + let mut index = 0_usize; + while index < bytes.len() { + if let Some(quote) = multiline { + if bytes[index..].starts_with(&[quote, quote, quote]) { + multiline = None; + index += 3; + continue; + } + } else if bytes[index..].starts_with(b"\"\"\"") { + multiline = Some(b'\"'); + index += 3; + continue; + } else if bytes[index..].starts_with(b"'''") { + multiline = Some(b'\''); + index += 3; + continue; + } else if bytes[index] == b'\n' { + return index > 0 && bytes[index - 1] == b'\r'; + } + index += 1; + } + false +} + +/// Converts document line terminators while leaving multiline-string content intact. +fn convert_document_lf_to_crlf(document: &str) -> String { + let mut output = String::with_capacity(document.len()); + let mut multiline: Option = None; + let mut chars = document.chars().peekable(); + while let Some(ch) = chars.next() { + if matches!(ch, '\"' | '\'') { + let mut probe = chars.clone(); + if probe.next() == Some(ch) && probe.next() == Some(ch) { + output.push(ch); + output.push(chars.next().expect("should have second quote")); + output.push(chars.next().expect("should have third quote")); + multiline = if multiline == Some(ch) { + None + } else if multiline.is_none() { + Some(ch) + } else { + multiline + }; + continue; + } + } + if ch == '\n' && multiline.is_none() && !output.ends_with('\r') { + output.push('\r'); + } + output.push(ch); + } + output } /// Strips a trailing inline `# comment` from a candidate table-header line. @@ -597,14 +650,6 @@ fn strip_inline_comment(line: &str) -> &str { } } -/// Whether `line` is exactly the `section_header` table header (for example -/// `[creative_opportunities]`), tolerating surrounding whitespace and a -/// trailing inline `# comment` — both valid TOML. -#[cfg(test)] -fn is_table_header(line: &str, section_header: &str) -> bool { - strip_inline_comment(line.trim()) == section_header -} - pub(super) fn replace_key_in_section( document: &str, section: &str, @@ -655,52 +700,6 @@ pub(super) fn replace_key_in_section( Ok(output) } -/// Sets `key` in `section`, replacing an existing assignment or inserting one. -/// -/// [`replace_key_in_section`] can only rewrite a key that is already present, so -/// it cannot add `section_root` or `section_segment` to a config that predates -/// them — which is every config a first templated run touches. This inserts -/// immediately after the section header instead, keeping the new key inside the -/// section's scalar block rather than stranding it after a subtable, where TOML -/// would read it as belonging to that subtable. -/// -/// # Errors -/// -/// Returns an error when `section` is not present in the document. -#[cfg(test)] -pub(super) fn upsert_key_in_section( - document: &str, - section: &str, - key: &str, - replacement_line: &str, -) -> CliResult { - if let Ok(replaced) = replace_key_in_section(document, section, key, replacement_line) { - return Ok(replaced); - } - - let section_header = format!("[{section}]"); - let Some(header_index) = document - .lines() - .position(|line| is_table_header(line, §ion_header)) - else { - return cli_error(format!( - "failed to update config because section `{section_header}` was not found" - )); - }; - - let mut lines: Vec = document.lines().map(str::to_string).collect(); - lines.insert(header_index + 1, replacement_line.to_string()); - - let mut output = lines.join("\n"); - if document.ends_with('\n') { - output.push('\n'); - } - if uses_crlf(document) { - output = output.replace("\r\n", "\n").replace('\n', "\r\n"); - } - Ok(output) -} - fn is_key_line(trimmed_line: &str, key: &str) -> bool { trimmed_line .strip_prefix(key) @@ -713,7 +712,7 @@ fn is_key_line(trimmed_line: &str, key: &str) -> bool { /// The existing id is kept only when a real merge preserves existing slots. /// On `--replace`, or when the config had no slots (e.g. a placeholder /// `[creative_opportunities]` section), the discovered id wins — mirroring -/// [`merge_slots`], which returns discovered-only in those cases. +/// the slot merge, which returns discovered-only in those cases. pub(super) fn resolve_network_id( existing: Option<&CreativeOpportunitiesConfig>, discovered_network_id: Option<&str>, @@ -1038,30 +1037,6 @@ slot_id = "sidebar" assert_eq!(creative["section_segment"].as_integer(), Some(0)); } - #[test] - fn upsert_keeps_an_inserted_key_inside_the_section_scalar_block() { - // Appending at the end of the section would land the key after a - // subtable, where TOML reads it as part of that subtable instead. - let document = "[creative_opportunities]\ngam_network_id = \"111\"\n\n\ - [[creative_opportunities.slot]]\nid = \"a\"\n\ - page_patterns = [\"/\"]\nformats = [{ width = 1, height = 1 }]\n"; - - let out = upsert_key_in_section( - document, - "creative_opportunities", - "section_root", - "section_root = \"homepage\"", - ) - .expect("should insert"); - - let value = toml::from_str::(&out).expect("valid TOML"); - assert_eq!( - value["creative_opportunities"]["section_root"].as_str(), - Some("homepage"), - "the key must belong to the section, not the slot subtable" - ); - } - #[test] fn splice_refuses_fresh_section_without_a_network_id() { // Reachable whenever the scraped unit path has no all-digit leading @@ -1176,6 +1151,38 @@ slot_id = "sidebar" ); } + #[test] + fn splice_does_not_infer_document_endings_from_multiline_string_content() { + let existing = "[publisher]\nother = \"\"\"a\r\nb\"\"\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice LF document"); + + assert!( + out.contains("[publisher]\nother"), + "an embedded CRLF must not convert document line endings" + ); + assert!( + out.contains("a\r\nb"), + "an unrelated multiline string value must remain byte-identical" + ); + } + + #[test] + fn splice_does_not_rewrite_bare_lf_inside_crlf_multiline_string() { + let existing = "[publisher]\r\nother = \"\"\"a\nb\"\"\"\r\n\r\n\ + [creative_opportunities]\r\ngam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + out.contains("a\nb"), + "a bare LF inside an unrelated multiline value must remain unchanged" + ); + } + #[test] fn render_slots_writes_non_finite_floor_price_as_valid_toml() { let slot = RenderSlot { @@ -1262,7 +1269,7 @@ slot_id = "sidebar" assert_eq!( out.lines() - .filter(|line| is_table_header(line, "[creative_opportunities]")) + .filter(|line| { strip_inline_comment(line.trim()) == "[creative_opportunities]" }) .count(), 1, "commented header must not be duplicated" diff --git a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs index 5516d9b97..ecf34d168 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs @@ -6,7 +6,7 @@ //! against inventory that does not exist, which is worse than a narrow literal. //! So this module is built to refuse rather than guess. //! -//! Three rules do the load-bearing work: +//! The inference applies three evidence rules: //! //! 1. **Positional binding.** `{network_id}` is bound to unit segment 0 and only //! if that segment is the resolved network id. Substring replacement would @@ -14,8 +14,8 @@ //! 2. **Exactly one varying segment.** Zero means nothing was proven and the //! path stays literal; two means the unit varies along a dimension the //! request path cannot supply (device, geo, experiment), so it is refused. -//! 3. **The witness rule.** Two pages must show *different* derived sections -//! *and* different unit segments. Without it a single-page crawl is +//! 3. **Cross-page variation.** Two pages must show *different* derived sections +//! and different unit segments. A single-page crawl is //! indistinguishable from a static path — literal, `{network_id}`-only and //! `{section}` all reproduce one observation equally well, and round-trip //! verification cannot tell them apart. Only variation can. @@ -146,7 +146,7 @@ pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> I .collect(); diagnostics.push(format!( "more than one section_segment ({}) explains the observed ad-unit paths \ - equally well, so no template can be chosen safely; keeping literal paths", + equally well, so no template can be chosen safely; slots without one safe literal path are omitted", indices.join(", ") )); None @@ -187,7 +187,7 @@ pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> I Err(reason) => { diagnostics.push(format!( "slot `{}` template `{template}` did not reproduce the observed \ - ad-unit paths ({reason}); keeping the literal path", + ad-unit paths ({reason}); refusing any unsafe fallback", slot.id )); literal_decision(slot) @@ -613,10 +613,10 @@ mod tests { } #[test] - fn a_slug_the_path_cannot_reproduce_stays_literal() { + fn a_slug_the_path_cannot_reproduce_is_refused() { // `/site-news` requests `.../sitenews`: the derived section and // the observed segment differ, so the template would render the wrong - // unit. Round-trip verification is what catches this. + // unit. Candidate analysis rejects the inconsistent section mapping. let table = table_for( "ad-header", &[ @@ -639,7 +639,7 @@ mod tests { } #[test] - fn an_unwitnessed_root_does_not_template() { + fn an_unwitnessed_root_is_refused() { // Every crawled page had a section, so `section_root` would be a guess // that silently mis-renders the homepage. let table = table_for( diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 0a534077c..715010532 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -14,7 +14,7 @@ pub mod page; use clap::{Args, Subcommand}; use crate::app_config::AppConfigArgs; -use crate::commands::audit::collector::BrowserOpts; +use crate::commands::audit::collector::{BrowserOpts, GenerateBrowserOpts}; use crate::commands::audit::page::PageAuditArgs; use crate::run::RunOutcome; @@ -167,7 +167,7 @@ pub(crate) struct AuditAdTemplatesGenerateArgs { #[arg(long, default_value_t = 750)] pub page_delay_ms: u64, #[command(flatten)] - pub browser: BrowserOpts, + pub browser: GenerateBrowserOpts, } impl AuditAdTemplatesGenerateArgs { @@ -249,7 +249,11 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result { } Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { gen_args.browser.validate()?; - let loaded = crate::app_config::load_file_settings(&gen_args.config)?; + let app_config_path = crate::app_config::resolve_app_config_file(&gen_args.config)?; + let raw_config = std::fs::read_to_string(&app_config_path).map_err(|error| { + format!("failed to read {}: {error}", app_config_path.display()) + })?; + let existing_creative = best_effort_creative_config(&raw_config); let profiles = gen_args.profiles()?; let collectors: Vec = profiles .iter() @@ -276,8 +280,8 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result { generate::run_update_slots( &generate::UpdateSlotsRequest { url: gen_args.url.as_str(), - config_path: &loaded.app_config_path, - existing_creative: loaded.settings.creative_opportunities.as_ref(), + config_path: &app_config_path, + existing_creative: existing_creative.as_ref(), page_patterns: &gen_args.page_patterns, replace: gen_args.replace, cookies: &gen_args.cookies, @@ -316,6 +320,15 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result { } } +fn best_effort_creative_config( + document: &str, +) -> Option { + toml::from_str::(document) + .ok() + .and_then(|value| value.get("creative_opportunities").cloned()) + .and_then(|value| value.try_into().ok()) +} + fn legacy_generate_args(args: &AuditArgs, url: &url::Url) -> generate::GenerateArgs { generate::GenerateArgs { url: url.to_string(), @@ -349,6 +362,17 @@ mod tests { assert!(value.is_empty(), "empty value should be allowed"); } + #[test] + fn invalid_baseline_still_yields_best_effort_creative_config() { + let document = "unknown_runtime_key = true\n\ + [creative_opportunities]\ngam_network_id = \"123\"\n"; + + let creative = best_effort_creative_config(document) + .expect("an unrelated invalid setting must not hide creative config"); + + assert_eq!(creative.gam_network_id, "123"); + } + #[test] fn parse_cookie_rejects_missing_equals() { let err = parse_cookie("datadome").expect_err("should reject missing `=`"); diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs index 14646168a..1b48aa1c6 100644 --- a/crates/trusted-server-cli/src/commands/config/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -77,7 +77,7 @@ pub struct AdTemplatesExplainArgs { /// Page path or full URL to evaluate. pub path_or_url: String, /// HTTP method to model. - #[arg(long, default_value = "GET")] + #[arg(long, default_value = "GET", value_parser = parse_http_method)] pub method: Method, /// Model a non-navigation request. #[arg(long)] @@ -93,6 +93,12 @@ pub struct AdTemplatesExplainArgs { pub consent_denied: bool, } +fn parse_http_method(raw: &str) -> Result { + let normalized = raw.to_ascii_uppercase(); + Method::from_bytes(normalized.as_bytes()) + .map_err(|error| format!("invalid HTTP method `{raw}`: {error}")) +} + /// Run an ad-template CLI command. /// /// # Errors @@ -145,7 +151,12 @@ fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), Strin plural(config.slot.len()) ) .map_err(output_error)?; - writeln!(out, "gam_network_id: {}", config.gam_network_id).map_err(output_error)?; + writeln!( + out, + "gam_network_id: {}", + escape_terminal_text(&config.gam_network_id) + ) + .map_err(output_error)?; writeln!( out, "auction_timeout_ms: {}", @@ -170,7 +181,7 @@ fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), Strin if loaded.settings.auction.providers.is_empty() { "(none)".to_string() } else { - loaded.settings.auction.providers.join(", ") + escape_terminal_text(&loaded.settings.auction.providers.join(", ")).into_owned() } ) .map_err(output_error)?; @@ -296,28 +307,27 @@ fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), let path = normalize_path_or_url(&args.path_or_url)?; writeln!(out, "path: {path}").map_err(output_error)?; - let Some(config) = &loaded.settings.creative_opportunities else { + let has_matches = if let Some(config) = &loaded.settings.creative_opportunities { + let matched = match_slots(&config.slot, &path); + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + true, + )?; + !matched.is_empty() + } else { writeln!(out, "creative_opportunities: not configured").map_err(output_error)?; - writeln!(out, "server-side ad stack: no").map_err(output_error)?; - return Ok(()); + false }; - let matched = match_slots(&config.slot, &path); - write_match_result( - out, - &path, - &matched, - &config.gam_network_id, - &config.section_for_path(&path), - true, - )?; - let method_pass = args.method == Method::GET; let navigation_pass = !args.non_navigation; let consent_pass = !args.consent_denied; let auction_enabled = loaded.settings.auction.enabled; let providers_configured = !loaded.settings.auction.providers.is_empty(); - let has_matches = !matched.is_empty(); let gate = evaluate_ad_stack_gate(AdStackGateInput { method_get: method_pass, @@ -389,16 +399,16 @@ fn write_match_result( details: bool, ) -> Result<(), String> { if matched.is_empty() { - writeln!(out, "{path}: no slots matched").map_err(output_error)?; + writeln!(out, "{}: no slots matched", escape_terminal_text(path)).map_err(output_error)?; return Ok(()); } let ids = matched .iter() - .map(|slot| slot.id.as_str()) + .map(|slot| escape_terminal_text(&slot.id).into_owned()) .collect::>() .join(", "); - writeln!(out, "{path}: matched {ids}").map_err(output_error)?; + writeln!(out, "{}: matched {ids}", escape_terminal_text(path)).map_err(output_error)?; if details { for slot in matched { @@ -433,10 +443,10 @@ fn format_slot(slot: &CreativeOpportunitySlot, gam_network_id: &str, section: &s .unwrap_or_else(|| "".to_string()); format!( "{} div={} gam={} patterns=[{}] formats=[{}] providers=[{}]", - slot.id, - slot.resolved_div_id(), - gam_unit_path, - slot.page_patterns.join(", "), + escape_terminal_text(&slot.id), + escape_terminal_text(slot.resolved_div_id()), + escape_terminal_text(&gam_unit_path), + escape_terminal_text(&slot.page_patterns.join(", ")), formats, providers, ) @@ -711,4 +721,13 @@ mod tests { assert_eq!(outcome, RunOutcome::AssertionFailed); } + + #[test] + fn http_method_parser_normalizes_standard_methods() { + assert_eq!( + parse_http_method("get").expect("should parse lowercase GET"), + Method::GET, + "lowercase GET must evaluate the same runtime gate as uppercase GET" + ); + } } diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 6ac5ad75e..9e8cd019c 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -438,23 +438,59 @@ mod tests { let Command::Audit(audit) = args.command else { panic!("expected audit command"); }; - let browser = match audit.command.expect("should parse audit subcommand") { - crate::commands::audit::AuditSubcommand::AdTemplates( - crate::commands::audit::AuditAdTemplatesCommand::Generate(args), - ) => args.browser, - crate::commands::audit::AuditSubcommand::AdTemplates( - crate::commands::audit::AuditAdTemplatesCommand::Verify(args), - ) => args.browser, - _ => panic!("expected ad-template mode"), - }; - assert_eq!(browser.chrome, Some(PathBuf::from("/tmp/test-chrome"))); - assert!(browser.headful); - assert!(browser.no_assume_consent); - assert_eq!(browser.browser_proxy.as_deref(), Some("127.0.0.1:8080")); - browser.validate().expect("should validate settle bounds"); + let (chrome, headful, no_assume_consent, browser_proxy, validation) = + match audit.command.expect("should parse audit subcommand") { + crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(args), + ) => { + let validation = args.browser.validate(); + ( + args.browser.chrome, + args.browser.headful, + args.browser.no_assume_consent, + args.browser.browser_proxy, + validation, + ) + } + crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Verify(args), + ) => { + let validation = args.browser.validate(); + ( + args.browser.chrome, + args.browser.headful, + args.browser.no_assume_consent, + args.browser.browser_proxy, + validation, + ) + } + _ => panic!("expected ad-template mode"), + }; + assert_eq!(chrome, Some(PathBuf::from("/tmp/test-chrome"))); + assert!(headful); + assert!(no_assume_consent); + assert_eq!(browser_proxy.as_deref(), Some("127.0.0.1:8080")); + validation.expect("should validate settle bounds"); } } + #[test] + fn audit_generate_does_not_expose_the_ignored_browser_profile_flag() { + assert!( + Args::try_parse_from([ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + "--browser-profile", + "mobile", + ]) + .is_err(), + "generation device selection must use --profiles" + ); + } + #[test] fn browser_settle_quiet_cannot_exceed_maximum() { let args = parse(&[ diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index e2ae84d15..aac688fed 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -554,14 +554,14 @@ impl CreativeOpportunitySlot { .iter() .filter_map(|pattern| match compile_page_pattern(pattern) { Ok(compiled) => Some(compiled), - Err(_) => { + Err(error) => { // Build-time validation only requires *one* valid pattern // per slot, so a mixed valid/invalid set passes the build // with the bad pattern silently dropped here. Warn so the // operator can see the slot matches fewer pages than // configured. log::warn!( - "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", + "slot `{}`: dropping page pattern '{}': {error}", self.id, pattern ); @@ -834,15 +834,12 @@ pub struct PrebidSlotParams { /// This is the single definition of what the runtime accepts as a page glob: /// a direct [`Pattern::new`], falling back to the `**`→`*` rewrite that /// [`CreativeOpportunitySlot::compile_patterns`] and -/// [`matches_path`](CreativeOpportunitySlot::matches_path) apply. Tooling that -/// writes patterns into operator config validates them through this function so -/// it cannot persist a pattern the runtime would silently drop. +/// [`matches_path`](CreativeOpportunitySlot::matches_path) apply. /// /// # Errors /// /// Returns an error string when the pattern compiles neither directly nor after /// normalisation. -/// pub(crate) fn compile_page_pattern(pattern: &str) -> Result { Pattern::new(pattern) .or_else(|_| Pattern::new(&pattern.replace("**", "*"))) @@ -992,8 +989,7 @@ pub struct AdStackGateInput { } /// Result of [`evaluate_ad_stack_gate`]: the three-state expectation plus the -/// list of gates that blocked the stack (empty unless `expected` is -/// [`No`](RuntimeAdStackExpected::No)). +/// original inputs used to derive per-gate diagnostics on demand. #[derive(Debug, Clone, Eq, PartialEq)] pub struct AdStackGateResult { /// The three-state ad-stack expectation. diff --git a/docs/guide/cli.md b/docs/guide/cli.md index a9277cfc2..df498dfb0 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -87,7 +87,7 @@ without launching a browser: | `match [--details]` | List matching slots; `--details` includes divs, paths, formats/providers. | | `check --expected-slot ID` | Assert the exact matching slot set; repeat `--expected-slot`. | | `check --expect-no-slots` | Assert that no slots match. | -| `explain ` | Print every runtime ad-stack gate and its final yes/no/unknown verdict. | +| `explain ` | Print every runtime ad-stack gate and its final yes/no verdict. | `check --allow-extra-slots` permits matches beyond the repeated `--expected-slot` values. It conflicts with `--expect-no-slots`. @@ -233,8 +233,8 @@ exist, so the command prefers a narrow literal path over a plausible guess. | --------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | | Only one page was crawled | Literal path. One observation cannot distinguish a literal from a template. | | The ad unit never varied by section | Literal path. | -| A section's slug is not derivable from its URL (`/site-news` requesting `.../sitenews`) | Literal path; the round-trip check catches it. | -| No root page was seen, so `section_root` is unknown | Literal path rather than a guessed fallback. | +| A section's slug is not derivable from its URL (`/site-news` requesting `.../sitenews`) | The slot is omitted and the reason is reported. | +| No root page was seen, so `section_root` is unknown | The slot is omitted rather than writing a guessed fallback. | | Two path segments could both be the section | No template; the ambiguity is reported. | | The ad unit varies by device, geo, or anything the URL cannot supply | The refused slot is omitted and the reason is written as a note. | | Crawled pages report different GAM network ids | The run fails; the pages are not one property. | @@ -418,7 +418,8 @@ Browser-backed ad-template generation and verification share `--chrome`, `--danger-accept-invalid-certs`. Verification also accepts `--browser-profile desktop|mobile`; generation uses `--profiles desktop,mobile` to compare both profiles. `--cookie NAME=VALUE` is -repeatable and creates host-only, root-path cookies. The quiet settle window +repeatable and creates host-only, root-path cookies; HTTPS targets also mark +them Secure. Verification refuses cookies when URLs span multiple origins. The quiet settle window must not exceed the maximum. `ts audit` is not an EdgeZero adapter command. It has no `--adapter` option and diff --git a/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md index 462e06c26..3ef89c2ed 100644 --- a/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md +++ b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md @@ -571,7 +571,7 @@ Restore the `audit` no-`--adapter` parser test. Add a script contract that sets - [ ] **Step 2: Replace sensitive-looking fixtures and stale assertions** -Replace `88059007`, `autoblog`, `car-research`/`carresearch`, and distinctive div tokens introduced by this PR with `123456789`, `publisher`, `/site-news`/`sitenews`, and neutral `ex_...` values. Update comments to describe shapes rather than customers. +Replace sensitive or customer-shaped fixtures introduced by this PR with fictional network IDs, publisher names, URL shapes, and neutral div tokens. Update comments to describe shapes rather than customers. Correct all touched `expect` messages to start with `should`, remove redundant crate/file `dead_code` allowances and annotate only genuinely deferred fields, reorder `Audit`, simplify the Prebid query parser so keys—not substrings—are matched, and bind legacy URLs directly without an impossible `expect`. diff --git a/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md index 1e26509c0..98af3e176 100644 --- a/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md +++ b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md @@ -313,35 +313,31 @@ Size compatibility is defined for Phase 1 as follows: - If ad unit path and div match but no numeric size overlap exists, the slot is `partial`, not `confirmed`. - Configured `video` and `native` formats are not used for Phase 1 GPT size - confirmation. If a matched slot has only non-banner formats, the verifier - reports it as `partial` with an unsupported-format warning unless a later - phase defines video/native verification. -- Out-of-page GPT slots are not confirmed in Phase 1 because the current - server-side ad-template path is slot/div based. They are reported as warnings - when observed. + confirmation. A matched slot with only non-banner formats is `unconfirmable` + with an unsupported-format warning and does not fail `--strict`. +- A sizeless live GPT slot is `partial` when the config declares banner sizes, + because that is observable drift and must fail `--strict`. ### 5.5 APS Evidence -When a configured slot has `providers.aps.slot_id`, the collector records -`apstag.fetchBids` calls and compares configured slot IDs and sizes with the APS -payload. - -APS evidence is a provider-level signal. Missing or ambiguous APS evidence -creates a provider warning, but it does not by itself make an otherwise GPT- -confirmed slot fail `--strict` in Phase 1. +Phase 1 does not wrap or collect `apstag.fetchBids`: APS is server-side provider +configuration and client-side calls are neither required nor authoritative for +the runtime ad-template decision. ### 5.6 Statuses -| Status | Meaning | -| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `confirmed` | GPT evidence matches the configured GAM unit path, div resolution, and compatible sizes. | -| `partial` | The page has some evidence for the configured slot, but not enough to confirm it. This includes DOM-only evidence, GPT path/div matches with incompatible sizes, GPT path/div matches for unsupported non-banner-only configured formats, and other non-confirming GPT evidence. | -| `missing` | No DOM or GPT evidence confirms the configured slot. | +| Status | Meaning | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `confirmed` | GPT evidence matches the configured GAM unit path, div resolution, and compatible sizes. | +| `partial` | The page has some evidence for the configured slot, but not enough to confirm it. This includes DOM-only evidence, GPT path/div matches with incompatible sizes, GPT path/div matches for unsupported non-banner-only configured formats, and other non-confirming GPT evidence. | +| `missing` | No DOM or GPT evidence confirms the configured slot. | +| `unconfirmable` | The checker cannot evaluate the configured format with Phase 1 evidence, such as a non-banner-only slot. This is reported but does not fail strict mode. | In `--strict` mode: - `missing` fails. - `partial` fails. +- `unconfirmable` does not fail. Provider issues are not statuses. They are warnings attached to the slot result. For example, a slot can be `confirmed` and still carry a warning that configured @@ -594,7 +590,7 @@ Extra live evidence is structured: } ``` -Allowed `kind` values for Phase 1 are `dom`, `gpt`, and `aps`. +Allowed `kind` values for Phase 1 are `dom` and `gpt`. Strict-mode failures with page results use the same shape and set `ok` to `false`. Example partial slot: @@ -695,7 +691,7 @@ Browser verification fails when: - at least one page-level error occurs in a multi-URL run; - command output cannot be written; - `--strict` is set, runtime verification is not skipped by a known gate, and - at least one matched slot is missing or partial. + at least one matched slot is missing or partial. `unconfirmable` is excluded. Browser collection can still produce a page result with warnings when: @@ -703,9 +699,7 @@ Browser collection can still produce a page result with warnings when: - a navigation redirects before final URL matching; - scroll evidence is incomplete; - GPT is not loaded; -- APS is not observed; -- provider evidence is ambiguous; -- extra live DOM/GPT/APS ad-slot evidence has no matched configured slot; +- extra live DOM/GPT ad-slot evidence has no matched configured slot; - no slots match the URL. ## 10. Testing diff --git a/scripts/test-cli.sh b/scripts/test-cli.sh index 14b179809..b12bb4b1e 100755 --- a/scripts/test-cli.sh +++ b/scripts/test-cli.sh @@ -20,5 +20,14 @@ fi cargo test --package trusted-server-cli --target "$HOST_TARGET" export TS_AUDIT_BROWSER_TESTS=1 +AUDIT_BROWSER_TEST_FILTER="commands::audit::browser::tests::" +AUDIT_BROWSER_TEST_COUNT="$({ + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --list +} | awk '/: test$/ { count += 1 } END { print count + 0 }')" +if [ "$AUDIT_BROWSER_TEST_COUNT" -eq 0 ]; then + echo "No ignored browser audit fixtures matched $AUDIT_BROWSER_TEST_FILTER" >&2 + exit 1 +fi cargo test --package trusted-server-cli --target "$HOST_TARGET" \ - commands::audit::browser::tests:: -- --ignored --test-threads=1 + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --test-threads=1 From efad9c978334527f2022de151b6c1ab33bedd5eb Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 12:53:31 +0530 Subject: [PATCH 342/395] Polish ad-template generator output --- Cargo.lock | 1 + Cargo.toml | 1 + crates/trusted-server-cli/Cargo.toml | 1 + .../audit/generate/browser_collector.rs | 52 ++++++++++++++- .../src/commands/audit/generate/slot_toml.rs | 65 ++++++++++++++----- crates/trusted-server-cli/src/main.rs | 7 ++ 6 files changed, 109 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f6e90982..cb467cefa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5346,6 +5346,7 @@ dependencies = [ "tokio-rustls", "toml", "toml_edit", + "tracing", "trusted-server-core", "url", "webpki-roots", diff --git a/Cargo.toml b/Cargo.toml index 31b92923f..a7fd306c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,7 @@ tokio-rustls = "0.26" toml = "1.1" toml_edit = "0.23.10" tower = "0.4" +tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index ec429dd7b..20c454a70 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -30,6 +30,7 @@ serde_json = { workspace = true } similar = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } toml = { workspace = true } toml_edit = { workspace = true } trusted-server-core = { workspace = true } diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index 7a46a3e6c..5628e498f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -422,19 +422,42 @@ async fn with_browser( .await .map_err(|_| report_error("timed out closing browser after audit")) .and_then(|closed| { - closed.map_err(|error| { + closed.map(|_| ()).map_err(|error| { report_error(format!("failed to close browser after audit: {error}")) }) }); + // Reap the child even when the CDP close request failed or timed out. Give + // waiting its own budget so a slow close cannot consume the entire teardown + // window and leave chromiumoxide's drop handler to kill the process. + let wait_result = timeout(BROWSER_CLOSE_TIMEOUT, browser.wait()) + .await + .map_err(|_| report_error("timed out waiting for browser process to exit after audit")) + .and_then(|waited| { + waited.map(|_| ()).map_err(|error| { + report_error(format!( + "failed waiting for browser process to exit after audit: {error}" + )) + }) + }); handler_task.abort(); let _ = handler_task.await; - match (result, close_result) { + let teardown_result = combine_browser_teardown_results(close_result, wait_result); + + match (result, teardown_result) { (Ok(()), Ok(_)) => Ok(()), (Ok(()), Err(error)) | (Err(error), _) => Err(error), } } +/// Combines already-attempted browser teardown phases, preserving the first error. +fn combine_browser_teardown_results( + close_result: CliResult<()>, + wait_result: CliResult<()>, +) -> CliResult<()> { + close_result.and(wait_result) +} + /// Collects one page on an already-launched browser. /// /// `discover_sitemap` runs the `robots.txt`/sitemap fetch from inside this @@ -1075,6 +1098,31 @@ mod tests { assert!(candidates.contains(&"Google Chrome for Testing")); } + #[test] + fn browser_teardown_reports_close_error_before_wait_error() { + let result = combine_browser_teardown_results( + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve teardown error"), + "close failed", + "the close failure is the first teardown failure" + ); + } + + #[test] + fn browser_teardown_reports_wait_error_when_close_succeeds() { + let result = combine_browser_teardown_results(Ok(()), Err("wait failed".to_string())); + + assert_eq!( + result.expect_err("should preserve wait error"), + "wait failed", + "a wait failure must not be mislabeled as a close failure" + ); + } + fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { let mut request = HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index e2d614f4f..72f54d417 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -235,7 +235,8 @@ pub(super) fn merge_render_slots_with_diagnostics( }; format!( "configured slot `{}` with div_id prefix `{}` matched {} discovered divs \ - ({sample}{suffix}); review whether they are distinct placements", + ({sample}{suffix}); runtime can resolve this configured slot to at most one \ + active element, so review whether they are distinct placements", slot.id, slot.div_id.as_deref().unwrap_or_default(), divs.len(), @@ -305,25 +306,22 @@ pub(super) fn render_slots(slots: &[RenderSlot]) -> String { if let Some(path) = &slot.gam_unit_path { out.push_str(&format!("gam_unit_path = {}\n", toml_string(path))); } - let patterns = slot - .page_patterns - .iter() - .map(|pattern| toml_string(pattern)) - .collect::>() - .join(", "); - out.push_str(&format!("page_patterns = [{patterns}]\n")); - let formats = slot - .formats - .iter() - .map(|(width, height, media_type)| match media_type { + out.push_str("page_patterns = [\n"); + for pattern in &slot.page_patterns { + out.push_str(&format!(" {},\n", toml_string(pattern))); + } + out.push_str("]\n"); + out.push_str("formats = [\n"); + for (width, height, media_type) in &slot.formats { + let rendered = match media_type { Some(kind) => { format!("{{ width = {width}, height = {height}, media_type = \"{kind}\" }}") } None => format!("{{ width = {width}, height = {height} }}"), - }) - .collect::>() - .join(", "); - out.push_str(&format!("formats = [{formats}]\n")); + }; + out.push_str(&format!(" {rendered},\n")); + } + out.push_str("]\n"); if let Some(floor) = slot.floor_price { // `f64` Display prints `NaN`, which is not valid TOML (`nan` is); // normalize non-finite values so the spliced config stays parseable. @@ -1206,6 +1204,37 @@ slot_id = "sidebar" toml::from_str::(&rendered).expect("rendered slots are valid TOML"); } + #[test] + fn render_slots_formats_long_arrays_across_indented_lines() { + let slot = RenderSlot { + id: "header".to_string(), + div_id: Some("div-gpt-ad-header".to_string()), + gam_unit_path: Some("/222/homepage/header".to_string()), + page_patterns: vec!["/".to_string(), "/news".to_string(), "/news/*".to_string()], + formats: vec![(728, 90, None), (970, 250, None), (300, 250, None)], + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: None, + }; + + let rendered = render_slots(&[slot]); + + assert!( + rendered.contains("page_patterns = [\n \"/\",\n \"/news\",\n \"/news/*\",\n]\n"), + "page patterns should be readable one-per-line" + ); + assert!( + rendered.contains( + "formats = [\n { width = 728, height = 90 },\n \ + { width = 970, height = 250 },\n \ + { width = 300, height = 250 },\n]\n" + ), + "formats should be readable one-per-line" + ); + toml::from_str::(&rendered).expect("formatted slots are valid TOML"); + } + #[test] fn splice_creates_section_when_absent() { // Config with no [creative_opportunities] at all — generate should append it. @@ -1493,6 +1522,10 @@ slot_id = "sidebar" assert_eq!(diagnostics.len(), 1); assert!(diagnostics[0].contains("matched 2 discovered divs")); assert!(diagnostics[0].contains("ad-footer")); + assert!( + diagnostics[0].contains("runtime can resolve this configured slot to at most one"), + "diagnostic should explain the runtime consequence" + ); assert!(diagnostics[0].contains("ad-header")); } diff --git a/crates/trusted-server-cli/src/main.rs b/crates/trusted-server-cli/src/main.rs index 9cf72215a..0a325bd55 100644 --- a/crates/trusted-server-cli/src/main.rs +++ b/crates/trusted-server-cli/src/main.rs @@ -2,6 +2,13 @@ fn main() { use std::process; + // Dependencies such as chromiumoxide instrument their internals with + // `tracing`. Without a subscriber, tracing's log-compatibility fallback + // forwards tolerated CDP decode warnings into the CLI's user-facing logger. + // Trusted Server uses `log` for intentional operator output, so install a + // no-op tracing subscriber to keep dependency diagnostics out of stdout and + // stderr without changing the process-wide `log` level. + let _ = tracing::subscriber::set_global_default(tracing::subscriber::NoSubscriber::default()); edgezero_cli::init_cli_logger(); match trusted_server_cli::run_from_env() { Ok(outcome) if outcome.exit_code() != 0 => process::exit(outcome.exit_code()), From bd052b490e364cbc461a418f1007934cd661ce9b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 13:47:02 +0530 Subject: [PATCH 343/395] Show ad-template generation progress --- .../audit/generate/browser_collector.rs | 149 ++++++- .../src/commands/audit/generate/collector.rs | 191 ++++++++- .../src/commands/audit/generate/mod.rs | 398 +++++++++++++++--- ...6-08-19-ad-template-generation-progress.md | 167 ++++++++ ...-ad-template-generation-progress-design.md | 59 +++ scripts/test-cli.sh | 25 +- 6 files changed, 914 insertions(+), 75 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md create mode 100644 docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index 5628e498f..5e23b46cc 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -18,7 +18,7 @@ use crate::commands::audit::browser::{ use crate::commands::audit::collector::GenerateBrowserOpts; use crate::commands::audit::generate::collector::{ AuditCollector, CollectedGptSlot, CollectedLink, CollectedPage, CollectedRequest, - CollectedScriptTag, ControlFlow, PageSink, RootPlanner, + CollectedScriptTag, CollectionProgress, ControlFlow, PageSink, ProgressSink, RootPlanner, }; use crate::error::{CliResult, report_error}; @@ -215,6 +215,10 @@ impl BrowserAuditCollector { } } +fn ignore_collection_progress(_: CollectionProgress<'_>) -> CliResult<()> { + Ok(()) +} + impl AuditCollector for BrowserAuditCollector { fn collect_page( &self, @@ -233,11 +237,13 @@ impl AuditCollector for BrowserAuditCollector { let settings = self.session(); runtime.block_on(async { let mut collected = None; + let mut ignore_progress = ignore_collection_progress; with_browser( vec![target_url.clone()], cookies, settings, None, + &mut ignore_progress, &mut |_, result| { collected = Some(result); Ok(ControlFlow::Stop) @@ -252,6 +258,7 @@ impl AuditCollector for BrowserAuditCollector { &self, targets: &[Url], cookies: &[(String, String)], + on_progress: ProgressSink<'_>, on_page: PageSink<'_>, ) -> CliResult<()> { if targets.is_empty() { @@ -275,6 +282,7 @@ impl AuditCollector for BrowserAuditCollector { cookies, self.session(), None, + on_progress, &mut |url, result| { collected_pages.push((url.clone(), result)); Ok(ControlFlow::Continue) @@ -292,6 +300,7 @@ impl AuditCollector for BrowserAuditCollector { &self, root: &Url, cookies: &[(String, String)], + on_progress: ProgressSink<'_>, planner: RootPlanner<'_>, on_page: PageSink<'_>, ) -> CliResult<()> { @@ -309,6 +318,7 @@ impl AuditCollector for BrowserAuditCollector { cookies, self.session(), Some(planner), + on_progress, &mut |url, result| { collected_pages.push((url.clone(), result)); Ok(ControlFlow::Continue) @@ -336,6 +346,7 @@ async fn with_browser( cookies: &[(String, String)], settings: SessionSettings, mut root_planner: Option>, + on_progress: ProgressSink<'_>, sink: PageSink<'_>, ) -> CliResult<()> { let SessionSettings { @@ -367,6 +378,7 @@ async fn with_browser( }) .map_err(report_error)?; + on_progress(CollectionProgress::Launching)?; let (mut browser, mut handler) = Browser::launch(config).await.map_err(|error| { report_error(format!( "failed to launch Chrome/Chromium for audit: {error}" @@ -376,10 +388,23 @@ async fn with_browser( let handler_task = tokio::spawn(async move { while handler.next().await.is_some() {} }); // Sitemap discovery is a whole-site fact, so only the first target pays for it. - let mut result = Ok(()); + let mut result: CliResult<()> = Ok(()); let mut index = 0; while index < targets.len() { let target = targets[index].clone(); + let total = if index == 0 && root_planner.is_some() { + None + } else { + Some(targets.len()) + }; + if let Err(error) = on_progress(CollectionProgress::Loading { + current: index + 1, + total, + url: &target, + }) { + result = Err(error); + break; + } // Pace the crawl. Back-to-back navigations are both discourteous to the // origin and a signal bot protection scores against the session. if index > 0 && !page_delay.is_zero() { @@ -399,6 +424,10 @@ async fn with_browser( && let Some(planner) = root_planner.as_deref_mut() && let Ok(root_page) = &collected { + if let Err(error) = on_progress(CollectionProgress::Planning) { + result = Err(error); + break; + } match planner(&target, root_page) { Ok(planned) => targets.extend(planned), Err(error) => { @@ -418,6 +447,7 @@ async fn with_browser( index += 1; } + let finalization_result = on_progress(CollectionProgress::Finalizing); let close_result = timeout(BROWSER_CLOSE_TIMEOUT, browser.close()) .await .map_err(|_| report_error("timed out closing browser after audit")) @@ -442,20 +472,20 @@ async fn with_browser( handler_task.abort(); let _ = handler_task.await; - let teardown_result = combine_browser_teardown_results(close_result, wait_result); - - match (result, teardown_result) { - (Ok(()), Ok(_)) => Ok(()), - (Ok(()), Err(error)) | (Err(error), _) => Err(error), - } + combine_browser_run_results(result, finalization_result, close_result, wait_result) } -/// Combines already-attempted browser teardown phases, preserving the first error. -fn combine_browser_teardown_results( +/// Combines already-attempted browser phases, preserving the first error. +fn combine_browser_run_results( + run_result: CliResult<()>, + finalization_result: CliResult<()>, close_result: CliResult<()>, wait_result: CliResult<()>, ) -> CliResult<()> { - close_result.and(wait_result) + run_result + .and(finalization_result) + .and(close_result) + .and(wait_result) } /// Collects one page on an already-launched browser. @@ -1051,6 +1081,18 @@ mod tests { use super::*; + /// Skips optional local runs, but makes the scripted/CI contract fail loudly. + fn browser_fixture_available() -> bool { + if resolve_chrome(None).is_ok() { + return true; + } + assert!( + std::env::var_os("TS_AUDIT_BROWSER_TESTS").is_none(), + "TS_AUDIT_BROWSER_TESTS requires Chrome/Chromium; set CHROME to its executable" + ); + false + } + #[test] fn successful_navigation_status_allows_redirects_but_rejects_errors() { assert!(is_successful_navigation_status(200)); @@ -1099,8 +1141,10 @@ mod tests { } #[test] - fn browser_teardown_reports_close_error_before_wait_error() { - let result = combine_browser_teardown_results( + fn browser_run_reports_close_error_before_wait_error() { + let result = combine_browser_run_results( + Ok(()), + Ok(()), Err("close failed".to_string()), Err("wait failed".to_string()), ); @@ -1113,8 +1157,9 @@ mod tests { } #[test] - fn browser_teardown_reports_wait_error_when_close_succeeds() { - let result = combine_browser_teardown_results(Ok(()), Err("wait failed".to_string())); + fn browser_run_reports_wait_error_when_close_succeeds() { + let result = + combine_browser_run_results(Ok(()), Ok(()), Ok(()), Err("wait failed".to_string())); assert_eq!( result.expect_err("should preserve wait error"), @@ -1123,6 +1168,80 @@ mod tests { ); } + #[test] + fn browser_run_preserves_collection_error_over_later_failures() { + let result = combine_browser_run_results( + Err("collection failed".to_string()), + Err("finalization progress failed".to_string()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve first browser run error"), + "collection failed" + ); + } + + #[test] + fn browser_run_reports_finalization_progress_before_teardown_errors() { + let result = combine_browser_run_results( + Ok(()), + Err("finalization progress failed".to_string()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve finalization progress error"), + "finalization progress failed" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn progress_failure_still_finalizes_browser_session() { + if !browser_fixture_available() { + return; + } + + let collector = BrowserAuditCollector::default(); + let target = Url::parse("http://127.0.0.1:9/").expect("should parse fixture URL"); + let mut phases = Vec::new(); + let error = collector + .collect_pages( + &[target], + &[], + &mut |progress| match progress { + CollectionProgress::Launching => { + phases.push("launching"); + Ok(()) + } + CollectionProgress::Loading { .. } => { + phases.push("loading"); + Err(report_error("simulated progress failure")) + } + CollectionProgress::Planning => { + phases.push("planning"); + Ok(()) + } + CollectionProgress::Finalizing => { + phases.push("finalizing"); + Ok(()) + } + }, + &mut |_, _| panic!("page sink should not run after progress failure"), + ) + .expect_err("should return progress failure after browser teardown"); + + let rendered_error = format!("{error:?}"); + assert!( + rendered_error.contains("simulated progress failure"), + "should preserve progress failure, got {rendered_error}" + ); + assert_eq!(phases, ["launching", "loading", "finalizing"]); + } + fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { let mut request = HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs index 7b9d87d0b..9e760a500 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs @@ -3,6 +3,33 @@ use url::Url; use crate::error::CliResult; +/// A user-visible phase reached while collecting browser audit evidence. +#[derive(Debug, Clone, Copy)] +pub(crate) enum CollectionProgress<'a> { + /// The browser process is about to launch. + Launching, + /// A page navigation is about to begin. + Loading { + /// One-based position of this attempted page in the crawl. + current: usize, + /// Total pages when planning has completed, or `None` for the root. + total: Option, + /// Target page; renderers must omit credentials, query, and fragment. + url: &'a Url, + }, + /// Follow-up pages are being selected from the collected root page. + Planning, + /// The browser session is being closed and its process reaped. + Finalizing, +} + +/// Sink invoked synchronously when browser collection reaches a visible phase. +/// +/// Returning an error stops new collection work. An already-launched browser +/// must still be finalized, closed, and waited on before that error is returned. +pub(crate) type ProgressSink<'a> = + &'a mut dyn for<'event> FnMut(CollectionProgress<'event>) -> CliResult<()>; + /// Sink invoked once per collected page during a batch crawl. /// /// Receives the per-page outcome so a failed page can be folded into the run as @@ -53,9 +80,15 @@ pub(crate) trait AuditCollector { &self, targets: &[Url], cookies: &[(String, String)], + on_progress: ProgressSink<'_>, on_page: PageSink<'_>, ) -> CliResult<()> { - for target in targets { + for (index, target) in targets.iter().enumerate() { + on_progress(CollectionProgress::Loading { + current: index + 1, + total: Some(targets.len()), + url: target, + })?; let collected = self.collect_page(target, cookies); if on_page(target, collected)? == ControlFlow::Stop { break; @@ -73,15 +106,34 @@ pub(crate) trait AuditCollector { &self, root: &Url, cookies: &[(String, String)], + on_progress: ProgressSink<'_>, planner: RootPlanner<'_>, on_page: PageSink<'_>, ) -> CliResult<()> { + on_progress(CollectionProgress::Loading { + current: 1, + total: None, + url: root, + })?; let root_page = self.collect_page(root, cookies)?; + on_progress(CollectionProgress::Planning)?; let targets = planner(root, &root_page)?; if on_page(root, Ok(root_page))? == ControlFlow::Stop { return Ok(()); } - self.collect_pages(&targets, cookies, on_page) + let total = targets.len() + 1; + for (index, target) in targets.iter().enumerate() { + on_progress(CollectionProgress::Loading { + current: index + 2, + total: Some(total), + url: target, + })?; + let collected = self.collect_page(target, cookies); + if on_page(target, collected)? == ControlFlow::Stop { + break; + } + } + Ok(()) } } @@ -125,6 +177,141 @@ pub(crate) struct CollectedLink { pub(crate) in_nav: bool, } +#[cfg(test)] +mod tests { + use super::*; + use crate::error::{cli_error, report_error}; + + struct ProgressCollector; + + impl AuditCollector for ProgressCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + if target_url.path() == "/broken" { + return cli_error("simulated page failure"); + } + Ok(CollectedPage { + requested_url: target_url.to_string(), + final_url: target_url.to_string(), + page_title: None, + html: String::new(), + script_tags: Vec::new(), + network_requests: Vec::new(), + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + }) + } + } + + fn record_progress(event: CollectionProgress<'_>) -> String { + match event { + CollectionProgress::Launching => "launching".to_string(), + CollectionProgress::Loading { + current, + total, + url, + } => format!( + "loading:{current}/{}:{}", + total.map_or_else(|| "?".to_string(), |total| total.to_string()), + url.path() + ), + CollectionProgress::Planning => "planning".to_string(), + CollectionProgress::Finalizing => "finalizing".to_string(), + } + } + + #[test] + fn default_collect_site_reports_root_planning_and_offset_followups() { + let collector = ProgressCollector; + let root = Url::parse("https://publisher.example/").expect("should parse root URL"); + let news = Url::parse("https://publisher.example/news").expect("should parse news URL"); + let broken = + Url::parse("https://publisher.example/broken").expect("should parse broken URL"); + let mut events = Vec::new(); + let mut outcomes = Vec::new(); + + collector + .collect_site( + &root, + &[], + &mut |event| { + events.push(record_progress(event)); + Ok(()) + }, + &mut |_, _| Ok(vec![news.clone(), broken.clone()]), + &mut |url, result| { + outcomes.push((url.path().to_string(), result.is_ok())); + Ok(ControlFlow::Continue) + }, + ) + .expect("should collect site despite one page outcome failing"); + + assert_eq!( + events, + [ + "loading:1/?:/", + "planning", + "loading:2/3:/news", + "loading:3/3:/broken", + ] + ); + assert_eq!( + outcomes, + [ + ("/".to_string(), true), + ("/news".to_string(), true), + ("/broken".to_string(), false) + ] + ); + } + + #[test] + fn default_collect_pages_reports_a_fixed_total() { + let collector = ProgressCollector; + let targets = [ + Url::parse("https://publisher.example/").expect("should parse root URL"), + Url::parse("https://publisher.example/broken").expect("should parse broken URL"), + ]; + let mut events = Vec::new(); + + collector + .collect_pages( + &targets, + &[], + &mut |event| { + events.push(record_progress(event)); + Ok(()) + }, + &mut |_, _| Ok(ControlFlow::Continue), + ) + .expect("should deliver failed page as an outcome"); + + assert_eq!(events, ["loading:1/2:/", "loading:2/2:/broken"]); + } + + #[test] + fn default_collection_stops_when_progress_fails() { + let collector = ProgressCollector; + let targets = [Url::parse("https://publisher.example/").expect("should parse root URL")]; + + let error = collector + .collect_pages( + &targets, + &[], + &mut |_| Err(report_error("simulated progress failure")), + &mut |_, _| panic!("page sink should not run after progress failure"), + ) + .expect_err("should return progress failure"); + + assert!(format!("{error:?}").contains("simulated progress failure")); + } +} + /// A single slot read from the page's live GPT registry. #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] pub(crate) struct CollectedGptSlot { diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 4b506939d..e44dd3e8b 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -543,39 +543,54 @@ pub(crate) fn run_update_slots( let mut planned = None; let mut fold_error = None; - first_collector.collect_site( - &target_url, - request.cookies, - &mut |_, root| { - root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); - if root_url.origin() != target_url.origin() { - return cli_error(format!( - "refusing cross-origin root redirect from {} to {}; the requested origin is the audit and cookie trust boundary", - target_url, root_url - )); - } - let plan = - crawl_plan::plan_crawl(&root_url, &root.links, &root.sitemap_locs, request.budget); - let targets = plan.targets(); - planned = Some(plan); - Ok(targets) - }, - &mut |url, collected| { - match collected { - Ok(page) => { - let final_url = page.final_url().unwrap_or_else(|_| url.clone()); - if let Err(error) = fold_collected(&mut table, &final_url, &page, &mut notes) { - fold_error = Some(error); - return Ok(collector::ControlFlow::Stop); - } + { + let mut progress_writer = CollectionProgressWriter { + out: err, + profile_label: first_label, + }; + let mut report_progress = + |progress: collector::CollectionProgress<'_>| progress_writer.write(progress); + first_collector.collect_site( + &target_url, + request.cookies, + &mut report_progress, + &mut |_, root| { + root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); + if root_url.origin() != target_url.origin() { + return cli_error(format!( + "refusing cross-origin root redirect from {} to {}; the requested origin is the audit and cookie trust boundary", + target_url, root_url + )); } - Err(error) => { - notes.push(format!("skipped `{url}` on {first_label}: {error}")); + let plan = crawl_plan::plan_crawl( + &root_url, + &root.links, + &root.sitemap_locs, + request.budget, + ); + let targets = plan.targets(); + planned = Some(plan); + Ok(targets) + }, + &mut |url, collected| { + match collected { + Ok(page) => { + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + if let Err(error) = + fold_collected(&mut table, &final_url, &page, &mut notes) + { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => { + notes.push(format!("skipped `{url}` on {first_label}: {error}")); + } } - } - Ok(collector::ControlFlow::Continue) - }, - )?; + Ok(collector::ControlFlow::Continue) + }, + )?; + } if let Some(error) = fold_error { return Err(error); } @@ -590,6 +605,10 @@ pub(crate) fn run_update_slots( // disagree about a slot's ad-unit path, that shows up as two observations of // one page, which inference already refuses to represent. for (label, collector) in collectors.iter().skip(1) { + let mut progress_writer = CollectionProgressWriter { + out: err, + profile_label: label, + }; let successful_pages = crawl_sections( *collector, &root_url, @@ -597,7 +616,7 @@ pub(crate) fn run_update_slots( request.cookies, &mut table, &mut notes, - label, + &mut progress_writer, )?; if successful_pages == 0 { return cli_error(format!( @@ -837,6 +856,54 @@ fn emit_notes(out: &mut dyn Write, notes: &mut Vec) -> CliResult<()> { Ok(()) } +/// Writes one immediately visible, profile-aware crawl progress line. +fn write_collection_progress( + out: &mut dyn Write, + profile_label: &str, + progress: collector::CollectionProgress<'_>, +) -> CliResult<()> { + let line = match progress { + collector::CollectionProgress::Launching => { + format!("Auditing {profile_label}: launching browser") + } + collector::CollectionProgress::Loading { + current, + total, + url, + } => { + let path = if url.path().is_empty() { + "/" + } else { + url.path() + }; + let path = crate::ad_templates::output::escape_terminal_text(path); + let total = total.map_or_else(|| "?".to_string(), |total| total.to_string()); + format!("Auditing {profile_label} [{current}/{total}]: {path}") + } + collector::CollectionProgress::Planning => { + format!("Auditing {profile_label}: planning site crawl") + } + collector::CollectionProgress::Finalizing => { + format!("Auditing {profile_label}: finalizing browser session") + } + }; + writeln!(out, "{line}") + .map_err(|error| report_error(format!("failed to write audit progress: {error}")))?; + out.flush() + .map_err(|error| report_error(format!("failed to flush audit progress: {error}"))) +} + +struct CollectionProgressWriter<'a> { + out: &'a mut dyn Write, + profile_label: &'a str, +} + +impl CollectionProgressWriter<'_> { + fn write(&mut self, progress: collector::CollectionProgress<'_>) -> CliResult<()> { + write_collection_progress(self.out, self.profile_label, progress) + } +} + /// Discovers a collected page's slots and folds them into `table`. /// /// Per-page collector warnings are appended to `notes`. They carry the reason a @@ -886,7 +953,7 @@ fn crawl_sections( cookies: &[(String, String)], table: &mut evidence::EvidenceTable, notes: &mut Vec, - profile_label: &str, + progress_writer: &mut CollectionProgressWriter<'_>, ) -> CliResult { let additional_targets = plan.targets(); if additional_targets.is_empty() { @@ -904,20 +971,32 @@ fn crawl_sections( let mut fold_error = None; let mut successful_pages = 0_usize; - collector.collect_pages(&targets, cookies, &mut |url, collected| { - match collected { - Ok(page) => { - successful_pages += 1; - let final_url = page.final_url().unwrap_or_else(|_| url.clone()); - if let Err(error) = fold_collected(table, &final_url, &page, notes) { - fold_error = Some(error); - return Ok(collector::ControlFlow::Stop); + { + let profile_label = progress_writer.profile_label; + let mut report_progress = + |progress: collector::CollectionProgress<'_>| progress_writer.write(progress); + collector.collect_pages( + &targets, + cookies, + &mut report_progress, + &mut |url, collected| { + match collected { + Ok(page) => { + successful_pages += 1; + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + if let Err(error) = fold_collected(table, &final_url, &page, notes) { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => { + notes.push(format!("skipped `{url}` on {profile_label}: {error}")); + } } - } - Err(error) => notes.push(format!("skipped `{url}` on {profile_label}: {error}")), - } - Ok(collector::ControlFlow::Continue) - })?; + Ok(collector::ControlFlow::Continue) + }, + )?; + } match fold_error { Some(error) => Err(error), None => Ok(successful_pages), @@ -1055,7 +1134,9 @@ fn validate_page_patterns(patterns: &[String]) -> CliResult<()> { #[cfg(test)] mod tests { - use std::cell::Cell; + use std::cell::{Cell, RefCell}; + use std::io; + use std::rc::Rc; use tempfile::TempDir; @@ -1117,6 +1198,210 @@ mod tests { struct FailingCollector; + #[derive(Clone, Default)] + struct SharedProgressState { + bytes: Rc>>, + flushes: Rc>, + } + + struct SharedProgressWriter { + state: SharedProgressState, + } + + impl Write for SharedProgressWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.state.bytes.borrow_mut().extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.state.flushes.set(self.state.flushes.get() + 1); + Ok(()) + } + } + + struct ObservingProgressCollector { + collected: CollectedPage, + state: SharedProgressState, + saw_flushed_progress: Cell, + } + + impl AuditCollector for ObservingProgressCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + Ok(self.collected.clone()) + } + + fn collect_site( + &self, + root: &Url, + _cookies: &[(String, String)], + on_progress: collector::ProgressSink<'_>, + planner: collector::RootPlanner<'_>, + on_page: collector::PageSink<'_>, + ) -> CliResult<()> { + on_progress(collector::CollectionProgress::Loading { + current: 1, + total: None, + url: root, + })?; + self.saw_flushed_progress + .set(!self.state.bytes.borrow().is_empty() && self.state.flushes.get() > 0); + on_progress(collector::CollectionProgress::Planning)?; + let _ = planner(root, &self.collected)?; + let _ = on_page(root, Ok(self.collected.clone()))?; + Ok(()) + } + } + + #[derive(Default)] + struct ProgressWriter { + bytes: Vec, + flushes: usize, + fail_write: bool, + fail_flush: bool, + } + + impl Write for ProgressWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + if self.fail_write { + return Err(io::Error::other("simulated progress write failure")); + } + self.bytes.extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.flushes += 1; + if self.fail_flush { + return Err(io::Error::other("simulated progress flush failure")); + } + Ok(()) + } + } + + #[test] + fn progress_lines_are_profile_aware_and_flush_immediately() { + let url = + Url::parse("https://user:pass@publisher.example/news\u{1b}[31m?token=secret#fragment") + .expect("should parse progress URL"); + let mut writer = ProgressWriter::default(); + + for progress in [ + collector::CollectionProgress::Launching, + collector::CollectionProgress::Loading { + current: 1, + total: None, + url: &url, + }, + collector::CollectionProgress::Planning, + collector::CollectionProgress::Loading { + current: 2, + total: Some(17), + url: &url, + }, + collector::CollectionProgress::Finalizing, + ] { + write_collection_progress(&mut writer, "desktop", progress) + .expect("should write progress"); + } + + let rendered = String::from_utf8(writer.bytes).expect("should render UTF-8 progress"); + assert_eq!( + rendered, + "Auditing desktop: launching browser\n\ + Auditing desktop [1/?]: /news%1B[31m\n\ + Auditing desktop: planning site crawl\n\ + Auditing desktop [2/17]: /news%1B[31m\n\ + Auditing desktop: finalizing browser session\n" + ); + assert_eq!(writer.flushes, 5, "should flush every progress line"); + assert!(!rendered.contains("user"), "should omit URL userinfo"); + assert!(!rendered.contains("secret"), "should omit URL query values"); + assert!(!rendered.contains("fragment"), "should omit URL fragments"); + assert!( + !rendered.contains('\u{1b}'), + "should not emit terminal escapes" + ); + } + + #[test] + fn progress_write_and_flush_failures_are_reported() { + let mut write_failure = ProgressWriter { + fail_write: true, + ..ProgressWriter::default() + }; + let write_error = write_collection_progress( + &mut write_failure, + "desktop", + collector::CollectionProgress::Launching, + ) + .expect_err("should report progress write failure"); + assert!(format!("{write_error:?}").contains("failed to write audit progress")); + + let mut flush_failure = ProgressWriter { + fail_flush: true, + ..ProgressWriter::default() + }; + let flush_error = write_collection_progress( + &mut flush_failure, + "desktop", + collector::CollectionProgress::Finalizing, + ) + .expect_err("should report progress flush failure"); + assert!(format!("{flush_error:?}").contains("failed to flush audit progress")); + } + + #[test] + fn update_slots_flushes_progress_before_collection_returns() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let state = SharedProgressState::default(); + let collector = ObservingProgressCollector { + collected: collected_page_with_header_slot(), + state: state.clone(), + saw_flushed_progress: Cell::new(false), + }; + let mut progress_writer = SharedProgressWriter { state }; + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut progress_writer, + ) + .expect("should generate slots"); + + assert!( + collector.saw_flushed_progress.get(), + "collector should observe flushed progress before returning" + ); + assert!( + !String::from_utf8(out) + .expect("should write UTF-8 output") + .contains("Auditing "), + "stdout should not contain progress" + ); + } + impl AuditCollector for FailingCollector { fn collect_page( &self, @@ -2046,8 +2331,25 @@ mod tests { .expect_err("an all-refused crawl must not write an empty slot array"); assert!(format!("{error:?}").contains("zero generated slots")); + let progress = String::from_utf8_lossy(&err); + for expected in [ + "Auditing desktop [1/?]: /", + "Auditing desktop: planning site crawl", + "Auditing desktop [2/2]: /news", + "Auditing mobile [1/2]: /", + "Auditing mobile [2/2]: /news", + ] { + assert!( + progress.contains(expected), + "should report `{expected}` while crawling, got:\n{progress}" + ); + } + assert!( + !String::from_utf8_lossy(&out).contains("Auditing "), + "progress must remain on stderr" + ); assert!( - String::from_utf8_lossy(&err).contains("skipped refused slot"), + progress.contains("skipped refused slot"), "the refusal reason should be reported" ); assert_eq!( diff --git a/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md b/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md new file mode 100644 index 000000000..ca4245314 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md @@ -0,0 +1,167 @@ +# Ad-template Generation Progress Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Show immediate, safe, profile-aware progress while `ts audit ad-templates generate` performs a long browser crawl. + +**Architecture:** Add typed progress events to the `AuditCollector` boundary so the browser can report work before buffered page results are returned. Render and flush those events from `run_update_slots` on stderr, using only URL paths. Preserve crawl/progress errors over teardown errors while always closing and waiting for Chrome. + +**Tech Stack:** Rust 2024, `std::io::Write`, existing `url`, `tokio`, `chromiumoxide`, and CLI test helpers; no new dependency. + +--- + +### Task 1: Define and render safe progress events + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing renderer and writer tests** + +Add tests in `generate/mod.rs` for events covering launch, `1/?`, `2/17`, and finalization. Assert that `https://user:pass@publisher.example/news?token=secret#fragment` renders only `/news`, terminal control bytes are escaped, stdout remains untouched, and a counting writer records an explicit `flush()`. Add writers that fail independently on `write()` and `flush()` and assert a CLI output error. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin progress -- --nocapture +``` + +Expected: FAIL because the progress event and renderer do not exist. + +- [ ] **Step 3: Add the progress model and renderer** + +In `collector.rs`, define a small event enum and callback type: + +```rust +pub(crate) enum CollectionProgress<'a> { + Launching, + Loading { + current: usize, + total: Option, + url: &'a Url, + }, + Planning, + Finalizing, +} + +pub(crate) type ProgressSink<'a> = + &'a mut dyn FnMut(CollectionProgress<'_>) -> CliResult<()>; +``` + +Add concise doc comments to the enum, every variant, and the callback alias. The +callback documentation must state that returning an error stops new collection +work but does not bypass an already-launched browser's finalization/close/wait. + +In `generate/mod.rs`, add a `write_collection_progress` helper that accepts a profile label, formats only `url.path()` (or `/` when empty), sanitizes it with `escape_terminal_text`, writes one line to stderr, and immediately calls `flush()`. Render and test `Planning` between the root load and subsequent page loads. + +- [ ] **Step 4: Run the focused tests and confirm GREEN** + +Run the command from Step 2. Expected: all progress renderer/writer tests pass. + +### Task 2: Propagate progress through collectors with teardown-safe failures + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `scripts/test-cli.sh` + +- [ ] **Step 1: Write failing collector tests** + +Test default `collect_pages` and `collect_site` count semantics, including an attempted page whose collection fails. The exact dynamic-site sequence is root `1/?`, planning, then follow-ups `2/total` through `total/total`; totals include the root and failed attempts advance the count. Add a Chrome-backed test whose progress callback fails during collection. It must return the progress error only after the browser teardown path completes. Extend the existing result-combination unit tests to cover first-error preservation across a collection/planning error, a later finalization-progress error, close error, and wait error, while proving finalization, close, and wait were all attempted. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::browser_collector::tests -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::collector::tests -- --nocapture +``` + +Expected: FAIL because collectors do not accept or emit progress callbacks. + +- [ ] **Step 3: Add callbacks to the collector boundary** + +Extend `collect_pages` and `collect_site` with `ProgressSink`. Default collectors emit `Loading` before each page. The root of a dynamically planned site emits `current: 1, total: None`, followed by `Planning`; after planning, default `collect_site` iterates follow-ups itself with an explicit offset so they report `2/total` onward. Fixed batches emit totals including the root, and failed attempts still consume their position. + +Pass the callback into `with_browser`. Adapt `BrowserAuditCollector::collect_page` with an explicit no-op progress sink because single-page artifact generation has no command progress writer. Emit `Launching` before browser launch, `Loading` immediately before each navigation, `Planning` immediately before invoking the root planner, and `Finalizing` before close/wait. Track only the first crawl/progress error: on callback failure, stop scheduling pages, still attempt finalization, `browser.close()`, and `browser.wait()`, then return that first error ahead of teardown errors. + +Extend `scripts/test-cli.sh` with a second ignored-test filter for +`commands::audit::generate::browser_collector::tests::` so the new Chrome-backed +progress-failure test is actually executed under `TS_AUDIT_BROWSER_TESTS=1` and +single-threaded, alongside the existing three browser audit fixtures. + +- [ ] **Step 4: Run unit and Chrome-backed tests and confirm GREEN** + +Run the focused command, then: + +```bash +./scripts/test-cli.sh +``` + +Expected: collector unit tests and all four Chrome-backed tests pass. + +### Task 3: Wire profile-aware progress into generation + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing generation tests** + +Update `run_update_slots` tests to assert the stderr buffer contains progress for the first profile's `1/?` root, planning, later known totals, the second profile's `1/total` root, and finalization. Assert dry-run diff/success output on stdout contains no progress lines. Add an ordering test with a shared observable writer and fake collector: from inside `collect_site`, after invoking and flushing the progress callback but before returning, assert the progress bytes are already visible. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin update_slots -- --nocapture +``` + +Expected: FAIL because `run_update_slots` does not provide progress callbacks. + +- [ ] **Step 3: Connect callbacks and profiles** + +Create a progress closure for the first profile and pass it to `collect_site`. Pass `err` through `crawl_sections`, create a closure for each later profile, and pass it to `collect_pages`. Keep notes and final summary behavior unchanged. + +- [ ] **Step 4: Run the focused tests and confirm GREEN** + +Run the command from Step 2. Expected: all generation tests pass and progress appears only in stderr. + +### Task 4: Verify and ship + +**Files:** + +- Verify all modified files plus the two design documents. + +- [ ] **Step 1: Format and lint** + +```bash +cargo fmt --all -- --check +cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings +git diff --check +cd docs && npm run format +``` + +Expected: all commands exit 0. + +- [ ] **Step 2: Run the complete local CLI suite** + +```bash +./scripts/test-cli.sh +``` + +Expected: unit, config, proxy, documentation, and Chrome-backed tests pass. + +- [ ] **Step 3: Review the scoped diff** + +Confirm no cookie values, real publisher data, or changes to the pre-existing `fastly.toml` modification are included. Request an independent code review and address concrete findings. + +- [ ] **Step 4: Commit and push** + +Stage only the progress implementation and its design/plan documents. Commit with `Show ad-template generation progress`, push `feature/ts-cli-ad-templates`, and confirm local HEAD matches the remote branch. diff --git a/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md new file mode 100644 index 000000000..8f75d0dcf --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md @@ -0,0 +1,59 @@ +# Ad-template generation progress design + +## Problem + +`ts audit ad-templates generate` audits up to the configured page budget for +each selected device profile (17 pages by default). Navigation and page settling +are intentionally bounded but can still take tens of seconds per page. The +browser collector buffers page results until the browser session closes, so the +command currently emits no output during most of that work and appears stuck. + +## Design + +Emit line-oriented progress on stderr while collection is running. Progress +must identify the device profile, current page, known total, and safe page +location. It must also identify non-page phases where a noticeable pause can +occur: launching the browser, planning the crawl after the root page, and +finalizing the browser session. + +Progress is an explicit collector callback rather than direct terminal output +inside the browser implementation. This keeps output policy in the command +layer, makes the behavior testable with in-memory writers, and lets non-browser +collectors preserve the same contract. Each line is flushed immediately. + +The first profile's root navigation has no final total because follow-up pages +are planned from the rendered root. It is reported as `1/?`; once planning +finishes, subsequent pages use a stable `current/total` count. Later profiles +receive the complete target list and report the root as `1/total`. Totals include +the root, and every attempted page advances the current count even if collection +fails. + +Progress never prints a full URL. It renders only the origin-free path, omitting +userinfo, query, and fragment data, then applies the CLI's existing terminal-text +sanitizer. An empty path is rendered as `/`. + +Stdout remains reserved for the generated diff or success summary. This keeps +`--dry-run` and shell redirection stable. Progress is intentionally plain text, +not an animated spinner, so it remains useful in logs and does not add a terminal +UI dependency. + +## Error handling + +Failure to write or flush progress is returned as a normal CLI output error. A +callback failure during a browser session stops further collection but does not +skip finalization, browser close, or process wait. An earlier collection or +planning error takes precedence over a later progress error; either takes +precedence over teardown errors. Close and wait are still attempted +independently. No cookie values, URL credentials, query values, fragments, or +browser credentials are included in progress. + +## Tests + +Unit tests will verify that progress is emitted before collection completes, +contains the specified profile-aware page counts, keeps stdout unchanged, +redacts URL credentials/query/fragment data, sanitizes paths, and reports +finalization. Writer tests will cover write failure, flush failure, and explicit +flush invocation. Collector tests will verify teardown still runs after progress +failure and that collection/planning errors, progress errors, and teardown errors +retain the stated precedence. The existing CLI and Chrome-backed suites will +verify the collector behavior and browser lifecycle remain intact. diff --git a/scripts/test-cli.sh b/scripts/test-cli.sh index b12bb4b1e..adc4ec0f0 100755 --- a/scripts/test-cli.sh +++ b/scripts/test-cli.sh @@ -20,14 +20,19 @@ fi cargo test --package trusted-server-cli --target "$HOST_TARGET" export TS_AUDIT_BROWSER_TESTS=1 -AUDIT_BROWSER_TEST_FILTER="commands::audit::browser::tests::" -AUDIT_BROWSER_TEST_COUNT="$({ +AUDIT_BROWSER_TEST_FILTERS=( + "commands::audit::browser::tests::" + "commands::audit::generate::browser_collector::tests::progress_failure_still_finalizes_browser_session" +) +for AUDIT_BROWSER_TEST_FILTER in "${AUDIT_BROWSER_TEST_FILTERS[@]}"; do + AUDIT_BROWSER_TEST_COUNT="$({ + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --list + } | awk '/: test$/ { count += 1 } END { print count + 0 }')" + if [ "$AUDIT_BROWSER_TEST_COUNT" -eq 0 ]; then + echo "No ignored browser audit fixtures matched $AUDIT_BROWSER_TEST_FILTER" >&2 + exit 1 + fi cargo test --package trusted-server-cli --target "$HOST_TARGET" \ - "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --list -} | awk '/: test$/ { count += 1 } END { print count + 0 }')" -if [ "$AUDIT_BROWSER_TEST_COUNT" -eq 0 ]; then - echo "No ignored browser audit fixtures matched $AUDIT_BROWSER_TEST_FILTER" >&2 - exit 1 -fi -cargo test --package trusted-server-cli --target "$HOST_TARGET" \ - "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --test-threads=1 + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --test-threads=1 +done From 1d4ac666a78462953c679bf15e483e502d041aa7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 14:02:04 +0530 Subject: [PATCH 344/395] Avoid duplicate crawl failure output --- .../audit/generate/browser_collector.rs | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index 5e23b46cc..99ba47728 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -1027,17 +1027,18 @@ async fn wait_for_page_settle( } fn validate_navigation_response(navigation_response: ArcHttpRequest) -> CliResult> { + // These failures are recoverable during a multi-page crawl. The caller + // records them once as a profile-aware `note:`; using `report_error` here + // would also log an unscoped duplicate in the middle of progress output. let request = navigation_response - .ok_or_else(|| report_error("browser audit did not capture the main document response"))?; + .ok_or_else(|| "browser audit did not capture the main document response".to_string())?; if let Some(failure_text) = &request.failure_text { - return Err(report_error(format!( - "main document request failed: {failure_text}" - ))); + return Err(format!("main document request failed: {failure_text}")); } let response = request.response.as_ref().ok_or_else(|| { - report_error("browser audit did not capture the main document HTTP response") + "browser audit did not capture the main document HTTP response".to_string() })?; if is_successful_navigation_status(response.status) { @@ -1117,6 +1118,21 @@ mod tests { ); } + #[test] + fn navigation_response_reports_chromium_request_failure() { + let mut request = + HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); + request.failure_text = Some("net::ERR_BLOCKED_BY_ORB".to_string()); + + let error = validate_navigation_response(Some(Arc::new(request))) + .expect_err("should reject Chromium request failures"); + + assert_eq!( + error, "main document request failed: net::ERR_BLOCKED_BY_ORB", + "the crawl should retain the browser failure for its final skipped-page note" + ); + } + #[test] fn resource_timing_buffer_warning_starts_at_threshold() { assert_eq!( From eb61c86c55511c272ffbfd04d88144eb99d91824 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 14:20:57 +0530 Subject: [PATCH 345/395] Plan volatile div collision refusal --- ...26-08-19-refuse-volatile-div-collisions.md | 63 +++++++++++++++++++ ...9-refuse-volatile-div-collisions-design.md | 55 ++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md create mode 100644 docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md diff --git a/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md new file mode 100644 index 000000000..e07b520de --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md @@ -0,0 +1,63 @@ +# Refuse Volatile Div-ID Collisions Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Prevent `ts audit ad-templates generate --replace` from writing exact per-render div IDs when several live elements normalize to one runtime prefix. + +**Architecture:** Keep collision detection in GPT discovery, where normalized and raw IDs are both available. On the first distinct collision, remove the tentatively accepted normalized slot and mark the group ambiguous; suppress all later members and emit one actionable diagnostic. Carry a separate evidence-present bit into `EvidenceTable` so collision-only pages are not classified as bot challenges. + +**Tech Stack:** Rust, Chromium GPT evidence model, built-in Rust test framework. + +--- + +### Task 1: Specify refusal behavior + +**Files:** +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` + +- [ ] Add assertions for documented `DiscoveredSlots::had_slot_evidence` and run one focused test to observe the expected missing-field compile failure. +- [ ] Add only the documented field with its derived/default false value so behavioral tests can compile; do not wire discovery or classification yet. +- [ ] Change the same-page collision test to require zero emitted slots and one refusal diagnostic. +- [ ] Assert the diagnostic names `ad-in_content`, explains that a broad prefix resolves only one element and raw IDs are volatile, and tells the operator to expose distinct stable IDs. +- [ ] Rename the test to `same_page_hex_normalization_collision_is_refused`. +- [ ] Extend `repeated_raw_div_after_a_normalization_collision_is_deduplicated` with repeats of both initial raw IDs and a third distinct ID; require zero slots and one diagnostic. +- [ ] Add `request_normalization_collision_is_refused`; require zero slots, one diagnostic with the same prefix/safety/action content, true evidence, and a surviving request-derived network ID. +- [ ] Add `ambiguous_registry_stem_still_suppresses_request_fallback` and require no slot resurrection. +- [ ] Require every registry/request collision test to assert `had_slot_evidence` is true. +- [ ] Add `collision_only_page_is_not_classified_as_empty` using a discovered collision result. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin same_page_hex_normalization_collision_is_refused` and confirm RED because two raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin repeated_raw_div_after_a_normalization_collision_is_deduplicated` and confirm RED because raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin request_normalization_collision_is_refused` and confirm RED because request-derived raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin ambiguous_registry_stem_still_suppresses_request_fallback` and confirm RED because the ambiguous registry group remains deployable. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin collision_only_page_is_not_classified_as_empty` and confirm RED because collision-only evidence is classified as empty. + +### Task 2: Refuse ambiguous collision groups + +**Files:** +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` + +- [ ] Replace raw-ID preservation with a collision result that distinguishes first ambiguity from later members. +- [ ] Remove the initially accepted normalized slot when ambiguity is first proven. +- [ ] Suppress the colliding and subsequent raw members. +- [ ] Emit one message naming the prefix, both unsafe representations, and the publisher-markup action. +- [ ] Set `had_slot_evidence` for any otherwise usable registry/request candidate and use it in empty-page classification. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin normalization_collision` and confirm the registry and request collision tests GREEN. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin ambiguous_registry_stem_still_suppresses_request_fallback` and confirm registry precedence GREEN. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin collision_only_page_is_not_classified_as_empty` and confirm GREEN. + +### Task 3: Verify and deliver + +**Files:** +- Verify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Verify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` +- Verify: `docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md` +- Verify: `docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md` + +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run `./scripts/test-cli.sh aarch64-apple-darwin`. +- [ ] Run `cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings`. +- [ ] Run `cd docs && npm run format`. +- [ ] Run `git diff --check` and inspect the scoped diff. +- [ ] Commit and push the fix to `feature/ts-cli-ad-templates`. diff --git a/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md new file mode 100644 index 000000000..60140d32e --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md @@ -0,0 +1,55 @@ +# Refuse Volatile Div-ID Collisions + +## Problem + +GPT discovery normalizes per-render div IDs such as +`ad-in_content--in_content-0` to the stable prefix `ad-in_content`. +When several live elements on the same page normalize to that prefix, the +runtime cannot represent them safely: one prefix resolves at most one element, +while each exact raw ID changes on a later render. The current collision path +preserves the raw IDs, causing `--replace` to write unusable literal slots. + +## Design + +Treat a source-local normalized collision as ambiguous and refuse the entire +group. The first observation remains tentatively accepted. When a second +distinct raw div ID normalizes to the same prefix, remove the first slot, record +the group as ambiguous, and suppress every later member. Emit one diagnostic +when the group first becomes ambiguous, naming the normalized prefix and +explaining that neither a single prefix nor volatile exact IDs are safe. Tell +the operator to expose distinct stable div IDs or prefixes in publisher markup +before configuring the placements. + +Registry and request-derived evidence retain separate collision maps, matching +the current source precedence: even an ambiguous registry stem continues to +suppress request fallback for that stem. Network-ID discovery is unaffected. + +`DiscoveredSlots` records whether any otherwise usable GPT slot evidence was +seen independently of how many safe slots remain. `EvidenceTable::fold_page` +uses that signal when classifying empty pages, so a collision-only page is not +mistaken for a bot challenge. Cross-page slot inference, merging, and +`--replace` otherwise remain unchanged because ambiguous slots never enter +those stages. + +## Safety and Output + +The generator prefers omission over a configuration that cannot match future +renders. For the observed Autoblog desktop crawl, replacement output should +therefore contain the stable `ad-header-0` and `ad-fixed_bottom-0` slots, while +the in-content collision group is explained in a note. The existing volatile +`rh-gam-kso` group and section-varying sidebar refusal remain unchanged. + +## Tests + +- A two-element same-page normalization collision yields no slots and one + diagnostic containing the prefix, both unsafe alternatives, and operator + action. +- Repeats of the first and second IDs plus a third distinct ID after a collision + remain suppressed and do not create additional diagnostics. +- Request-derived collisions follow the same policy. +- An ambiguous registry stem still suppresses request fallback, and network-ID + discovery survives when every collided slot is omitted. +- A collision-only page is recorded as having evidence rather than as an empty + challenge page. +- Existing normalization, request fallback, fragment detection, and full CLI + tests remain green. From daae07bc64373d69faf4034e31b5290a22cfe764 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 14:29:34 +0530 Subject: [PATCH 346/395] Refuse ambiguous volatile ad slots --- .../src/commands/audit/generate/evidence.rs | 23 ++- .../src/commands/audit/generate/gpt_slots.rs | 170 ++++++++++++------ .../src/commands/audit/generate/mod.rs | 49 ++++- ...26-08-19-refuse-volatile-div-collisions.md | 3 + 4 files changed, 191 insertions(+), 54 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs index 43bb5e6fa..78fd6d8a6 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -145,7 +145,7 @@ impl EvidenceTable { if let Some(network_id) = &discovered.gam_network_id { self.network_ids.insert(network_id.clone()); } - if discovered.slots.is_empty() { + if !discovered.had_slot_evidence { if !self.non_empty_pages.contains(path) { self.empty_pages.insert(path.to_string()); } @@ -619,6 +619,27 @@ mod tests { ); } + #[test] + fn collision_only_page_is_not_classified_as_empty() { + let discovered = page( + &[ + ("/123/site/home", "ad-x-aaaaaaaaaaaaaaaa-0", &[(300, 250)]), + ("/123/site/home", "ad-x-bbbbbbbbbbbbbbbb-1", &[(300, 250)]), + ], + false, + ); + let mut table = EvidenceTable::default(); + + table.fold_page("/collision-only", &discovered); + + assert!(discovered.had_slot_evidence); + assert!(discovered.slots.is_empty()); + assert!( + table.empty_pages().is_empty(), + "intentionally omitted GPT evidence must not look like a bot challenge" + ); + } + #[test] fn empty_table_resolves_no_network_id_rather_than_erroring() { let table = EvidenceTable::default(); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index af5eb4f2f..f670dad2b 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -85,6 +85,9 @@ pub(crate) struct DiscoveredSlot { pub(crate) struct DiscoveredSlots { /// GAM network id shared by the discovered slots, if any were found. pub(crate) gam_network_id: Option, + /// Whether the page exposed any otherwise usable slot evidence, including + /// ambiguous placements that were intentionally omitted from `slots`. + pub(crate) had_slot_evidence: bool, /// The reconstructed slots, deduplicated by div id in first-seen order. pub(crate) slots: Vec, /// Diagnostics for placements whose normalized stable stems collided. @@ -109,17 +112,18 @@ pub(crate) fn discover_gpt_slots( let mut slots = Vec::new(); let mut warnings = Vec::new(); let mut gam_network_id = None; + let mut had_slot_evidence = false; let mut registry_divs: BTreeMap> = BTreeMap::new(); for entry in registry { let Some(slot) = slot_from_registry(entry, page_has_prebid) else { continue; }; - if push_slot_preserving_collisions(&mut slots, &mut registry_divs, slot, &entry.div_id) { - warnings.push(format!( - "normalized div-id collision retained raw volatile id `{}`; it may not match a later render", - entry.div_id - )); + had_slot_evidence = true; + if let Some(prefix) = + push_slot_refusing_collisions(&mut slots, &mut registry_divs, slot, &entry.div_id) + { + warnings.push(ambiguous_collision_warning(&prefix)); } if gam_network_id.is_none() { gam_network_id = network_id_from_unit_path(&entry.gam_unit_path); @@ -132,13 +136,14 @@ pub(crate) fn discover_gpt_slots( let Some((network_id, slot, raw_div)) = parse_gampad_request(&request.url) else { continue; }; + had_slot_evidence = true; if registry_stems.contains(&slot.div_id) { continue; } - if push_slot_preserving_collisions(&mut slots, &mut request_divs, slot, &raw_div) { - warnings.push(format!( - "normalized request div-id collision retained raw volatile id `{raw_div}`; it may not match a later render" - )); + if let Some(prefix) = + push_slot_refusing_collisions(&mut slots, &mut request_divs, slot, &raw_div) + { + warnings.push(ambiguous_collision_warning(&prefix)); } if gam_network_id.is_none() { gam_network_id = Some(network_id); @@ -148,48 +153,54 @@ pub(crate) fn discover_gpt_slots( DiscoveredSlots { gam_network_id, + had_slot_evidence, slots, warnings, } } -/// Adds one source-local slot while retaining distinct raw div IDs that share a stem. +/// Adds one source-local slot unless distinct raw div IDs share its stable stem. /// -/// Returns whether a normalized collision forced raw, potentially volatile IDs -/// to be retained for both placements. -fn push_slot_preserving_collisions( +/// The first distinct collision removes the tentatively accepted slot and +/// returns its stem for one diagnostic. Repeats and later collision members stay +/// suppressed and return `None`. +fn push_slot_refusing_collisions( slots: &mut Vec, seen_divs: &mut BTreeMap>, - mut slot: DiscoveredSlot, + slot: DiscoveredSlot, raw_div: &str, -) -> bool { +) -> Option { let normalized = slot.div_id.clone(); let raw_div = raw_div.strip_suffix("-container").unwrap_or(raw_div); match seen_divs.get_mut(&normalized) { None => { seen_divs.insert(normalized, BTreeSet::from([raw_div.to_string()])); slots.push(slot); - false + None } - Some(raw_divs) if raw_divs.contains(raw_div) => false, + Some(raw_divs) if raw_divs.contains(raw_div) => None, Some(raw_divs) => { - let previous_raw = raw_divs - .first() - .expect("should have a first raw div after initial insertion") - .clone(); - if let Some(previous) = slots.iter_mut().find(|entry| entry.div_id == normalized) { - previous.div_id.clone_from(&previous_raw); - previous.id = slot_id_from_div(&previous_raw); - } - slot.div_id = raw_div.to_string(); - slot.id = slot_id_from_div(raw_div); + let became_ambiguous = raw_divs.len() == 1; raw_divs.insert(raw_div.to_string()); - slots.push(slot); - true + if became_ambiguous { + slots.retain(|entry| entry.div_id != normalized); + Some(normalized) + } else { + None + } } } } +fn ambiguous_collision_warning(prefix: &str) -> String { + format!( + "skipped ambiguous div-id prefix `{prefix}`: multiple active elements normalized to it, \ + but the runtime can resolve a prefix to only one active element and exact div ids change \ + across renders; expose distinct stable div ids in publisher markup before configuring \ + these placements" + ) +} + /// Converts a live-registry slot into a [`DiscoveredSlot`]. /// /// Returns `None` when the slot has no usable pixel size or its div id is a @@ -909,7 +920,7 @@ mod tests { } #[test] - fn same_page_hex_normalization_collision_retains_both_raw_slots() { + fn same_page_hex_normalization_collision_is_refused() { let registry = vec![ registry_slot( "/987654321/site/homepage", @@ -925,44 +936,81 @@ mod tests { let discovered = discover_gpt_slots(®istry, &[], false); - assert_eq!(discovered.slots.len(), 2); - assert_eq!( - discovered - .slots - .iter() - .map(|slot| slot.div_id.as_str()) - .collect::>(), - [ - "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", - "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-0", - ] - ); - assert_eq!(discovered.slots[0].formats, vec![(300, 250)]); - assert_ne!(discovered.slots[0].id, discovered.slots[1].id); - assert_eq!( - discovered.warnings.len(), - 1, - "writing raw volatile IDs must be diagnosable" + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "neither a broad prefix nor per-render exact IDs are safe" ); + assert_ambiguous_collision_warning(&discovered, "ad-in_content"); } #[test] fn repeated_raw_div_after_a_normalization_collision_is_deduplicated() { let first = "ad-x-aaaaaaaaaaaaaaaa-0"; let second = "ad-x-bbbbbbbbbbbbbbbb-1"; + let third = "ad-x-cccccccccccccccc-2"; let registry = vec![ registry_slot("/123456789/site/home", first, &[(300, 250)]), registry_slot("/123456789/site/home", second, &[(300, 250)]), + registry_slot("/123456789/site/home", first, &[(300, 250)]), registry_slot("/123456789/site/home", second, &[(300, 250)]), + registry_slot("/123456789/site/home", third, &[(300, 250)]), ]; let discovered = discover_gpt_slots(®istry, &[], false); - assert_eq!( - discovered.slots.len(), - 2, - "an exact raw div repeat must remain first-seen deduplicated" + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "no repeat or later collision member may resurrect the group" + ); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn request_normalization_collision_is_refused() { + let discovered = from_requests(&[ + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-aaaaaaaaaaaaaaaa-0&prev_iu_szs=300x250", + ), + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-bbbbbbbbbbbbbbbb-1&prev_iu_szs=300x250", + ), + ]); + + assert!(discovered.had_slot_evidence); + assert!(discovered.slots.is_empty()); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn ambiguous_registry_stem_still_suppresses_request_fallback() { + let registry = vec![ + registry_slot( + "/123456789/site/home", + "ad-x-aaaaaaaaaaaaaaaa-0", + &[(300, 250)], + ), + registry_slot( + "/123456789/site/home", + "ad-x-bbbbbbbbbbbbbbbb-1", + &[(300, 250)], + ), + ]; + let requests = vec![request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-cccccccccccccccc-2&prev_iu_szs=300x250", + )]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "request fallback must not resurrect an ambiguous registry stem" ); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_ambiguous_collision_warning(&discovered, "ad-x"); } #[test] @@ -984,4 +1032,22 @@ mod tests { "request fallback must not destabilize a registry-derived prefix" ); } + + fn assert_ambiguous_collision_warning(discovered: &DiscoveredSlots, prefix: &str) { + assert_eq!(discovered.warnings.len(), 1); + let warning = &discovered.warnings[0]; + assert!(warning.contains(prefix), "warning should name the prefix"); + assert!( + warning.contains("one active element"), + "warning should explain why the broad prefix is unsafe" + ); + assert!( + warning.contains("change across renders"), + "warning should explain why raw IDs are unsafe" + ); + assert!( + warning.contains("distinct stable div ids"), + "warning should tell the operator how to make the placements configurable" + ); + } } diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index e44dd3e8b..4cedacc5b 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -935,7 +935,11 @@ fn fold_collected( &collected.network_requests, page_has_prebid, ); - notes.extend(discovered.warnings.iter().cloned()); + for warning in &discovered.warnings { + if !notes.contains(warning) { + notes.push(warning.clone()); + } + } table.fold_page(url.path(), &discovered); Ok(()) } @@ -1498,6 +1502,25 @@ mod tests { collected } + fn collected_page_with_ambiguous_slots(url: &str) -> CollectedPage { + let mut collected = collected_page(); + collected.requested_url = url.to_string(); + collected.final_url = url.to_string(); + collected.gpt_slots = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/in-content".to_string(), + div_id: "ad-x-aaaaaaaaaaaaaaaa-0".to_string(), + sizes: vec![(300, 250)], + }, + collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/in-content".to_string(), + div_id: "ad-x-bbbbbbbbbbbbbbbb-1".to_string(), + sizes: vec![(300, 250)], + }, + ]; + collected + } + fn audit_args(url: &str) -> GenerateArgs { GenerateArgs { url: url.to_string(), @@ -1531,6 +1554,30 @@ mod tests { } } + #[test] + fn repeated_ambiguous_collision_note_is_emitted_once() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + for url in [ + "https://publisher.example/", + "https://publisher.example/news", + ] { + fold_collected( + &mut table, + &Url::parse(url).expect("should parse fixture URL"), + &collected_page_with_ambiguous_slots(url), + &mut notes, + ) + .expect("should fold ambiguous page evidence"); + } + + assert_eq!( + notes.len(), + 1, + "the same site-wide collision guidance should not repeat per page" + ); + } + #[test] fn merge_refuses_to_change_policy_used_by_preserved_templates() { let existing: CreativeOpportunitiesConfig = toml::from_str( diff --git a/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md index e07b520de..ff430323f 100644 --- a/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md +++ b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md @@ -13,6 +13,7 @@ ### Task 1: Specify refusal behavior **Files:** + - Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` - Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` @@ -35,6 +36,7 @@ ### Task 2: Refuse ambiguous collision groups **Files:** + - Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` - Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` @@ -50,6 +52,7 @@ ### Task 3: Verify and deliver **Files:** + - Verify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` - Verify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` - Verify: `docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md` From cc16ddddbcd4ff9ef8b7a8c02caf74d6dd4fa25c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 14:43:29 +0530 Subject: [PATCH 347/395] Refuse known volatile ad slot family --- .../src/commands/audit/generate/gpt_slots.rs | 126 +++++++++++++++++- ...26-08-19-refuse-volatile-div-collisions.md | 12 ++ ...9-refuse-volatile-div-collisions-design.md | 14 +- 3 files changed, 144 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index f670dad2b..a0ce98825 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -58,6 +58,10 @@ const GAMPAD_HOSTS: &[&str] = &["securepubads.g.doubleclick.net", "pubads.g.doub /// Common GPT div-id prefix stripped when deriving a slot id. const GPT_DIV_PREFIX: &str = "div-gpt-ad-"; +/// Publisher integration whose div IDs include a per-render timestamp/random +/// token before the placement kind (for example, `_ei_inarticle_1`). +const RH_GAM_KSO_PREFIX: &str = "rh-gam-kso"; + /// Minimum width/height for a format to be treated as a real creative size. /// /// GPT encodes fluid/native aspect-ratio markers (e.g. `4x1`, `8x1`) alongside @@ -120,14 +124,22 @@ pub(crate) fn discover_gpt_slots( continue; }; had_slot_evidence = true; + if gam_network_id.is_none() { + gam_network_id = network_id_from_unit_path(&entry.gam_unit_path); + } + if let Some(prefix) = known_per_render_div_prefix(&entry.div_id) { + registry_divs + .entry(slot.div_id.clone()) + .or_default() + .insert(entry.div_id.clone()); + push_unique_warning(&mut warnings, known_per_render_warning(prefix)); + continue; + } if let Some(prefix) = push_slot_refusing_collisions(&mut slots, &mut registry_divs, slot, &entry.div_id) { warnings.push(ambiguous_collision_warning(&prefix)); } - if gam_network_id.is_none() { - gam_network_id = network_id_from_unit_path(&entry.gam_unit_path); - } } let registry_stems: BTreeSet = registry_divs.keys().cloned().collect(); @@ -137,17 +149,21 @@ pub(crate) fn discover_gpt_slots( continue; }; had_slot_evidence = true; + if gam_network_id.is_none() { + gam_network_id = Some(network_id); + } if registry_stems.contains(&slot.div_id) { continue; } + if let Some(prefix) = known_per_render_div_prefix(&raw_div) { + push_unique_warning(&mut warnings, known_per_render_warning(prefix)); + continue; + } if let Some(prefix) = push_slot_refusing_collisions(&mut slots, &mut request_divs, slot, &raw_div) { warnings.push(ambiguous_collision_warning(&prefix)); } - if gam_network_id.is_none() { - gam_network_id = Some(network_id); - } } make_slot_ids_unique(&mut slots); @@ -201,6 +217,36 @@ fn ambiguous_collision_warning(prefix: &str) -> String { ) } +fn known_per_render_div_prefix(div_id: &str) -> Option<&'static str> { + let div_id = div_id.strip_suffix("-container").unwrap_or(div_id); + let remainder = div_id.strip_prefix("rh-gam-kso_")?; + let (token, placement) = remainder.split_once("_ei_")?; + let leading_digits = token.bytes().take_while(u8::is_ascii_digit).count(); + let token_is_dynamic = leading_digits >= 8 + && token.len() > leading_digits + && token.bytes().all(|byte| byte.is_ascii_alphanumeric()); + let placement_index = placement + .strip_prefix("inarticle_") + .or_else(|| placement.strip_prefix("overlay_"))?; + let placement_is_known = + !placement_index.is_empty() && placement_index.bytes().all(|byte| byte.is_ascii_digit()); + (token_is_dynamic && placement_is_known).then_some(RH_GAM_KSO_PREFIX) +} + +fn known_per_render_warning(prefix: &str) -> String { + format!( + "skipped known per-render div-id family `{prefix}`: exact div ids change across renders \ + and no distinct stable element prefix is available; expose distinct stable div ids in \ + publisher markup before configuring these placements" + ) +} + +fn push_unique_warning(warnings: &mut Vec, warning: String) { + if !warnings.contains(&warning) { + warnings.push(warning); + } +} + /// Converts a live-registry slot into a [`DiscoveredSlot`]. /// /// Returns `None` when the slot has no usable pixel size or its div id is a @@ -984,6 +1030,66 @@ mod tests { assert_ambiguous_collision_warning(&discovered, "ad-x"); } + #[test] + fn single_known_per_render_registry_slot_is_refused() { + let discovered = discover_gpt_slots( + &[registry_slot( + "/22558409563/autoblog.com_In-Article_Desktop_ESP_jfOmMslaux", + "rh-gam-kso_26332072TPTy2yC1wkhc_ei_inarticle_1", + &[(300, 250)], + )], + &[], + false, + ); + + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "one observation of a known per-render family must not be written literally" + ); + assert_eq!(discovered.gam_network_id.as_deref(), Some("22558409563")); + assert_known_per_render_warning(&discovered); + } + + #[test] + fn single_known_per_render_request_slot_is_refused() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=22558409563%2Cautoblog.com_In-Article_Desktop_ESP_jfOmMslaux&dids=rh-gam-kso_26332072TPTy2yC1wkhc_ei_inarticle_1&prev_iu_szs=300x250", + )]); + + assert!(discovered.had_slot_evidence); + assert!(discovered.slots.is_empty()); + assert_eq!(discovered.gam_network_id.as_deref(), Some("22558409563")); + assert_known_per_render_warning(&discovered); + } + + #[test] + fn known_per_render_match_does_not_claim_arbitrary_vendor_ids() { + assert_eq!( + known_per_render_div_prefix("rh-gam-kso_26332072TPTy2yC1wkhc_ei_inarticle_1"), + Some("rh-gam-kso") + ); + assert_eq!( + known_per_render_div_prefix("rh-gam-kso_26332072TPTy2yC1wkhc_ei_overlay_1-container"), + Some("rh-gam-kso") + ); + for stable in [ + "rh-gam-kso_stable_ei_inarticle_1", + "rh-gam-kso_26332072_ei_inarticle_1", + "rh-gam-kso_26332072TPTy2yC1wkhc_ei_sidebar_1", + "rh-gam-kso_26332072TPTy2yC1wkhc_ei_overlay_stable", + "rh-gam-kso_26332072TPTy2yC1wkhc_ei_overlay_", + "rh-gam-kso_26332072TPTy2yC1wkhc_ei_overlay_1_extra", + "rh-gam-kso-header", + ] { + assert_eq!( + known_per_render_div_prefix(stable), + None, + "`{stable}` should not match the narrow per-render family" + ); + } + } + #[test] fn ambiguous_registry_stem_still_suppresses_request_fallback() { let registry = vec![ @@ -1050,4 +1156,12 @@ mod tests { "warning should tell the operator how to make the placements configurable" ); } + + fn assert_known_per_render_warning(discovered: &DiscoveredSlots) { + assert_eq!(discovered.warnings.len(), 1); + let warning = &discovered.warnings[0]; + assert!(warning.contains("rh-gam-kso")); + assert!(warning.contains("change across renders")); + assert!(warning.contains("distinct stable div ids")); + } } diff --git a/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md index ff430323f..19a83a801 100644 --- a/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md +++ b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md @@ -64,3 +64,15 @@ - [ ] Run `cd docs && npm run format`. - [ ] Run `git diff --check` and inspect the scoped diff. - [ ] Commit and push the fix to `feature/ts-cli-ad-templates`. + +### Task 4: Refuse a known single-observation volatile family + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` + +- [ ] Add failing registry and request tests for a single `rh-gam-kso__ei_` observation. +- [ ] Add a narrow recognizer requiring the vendor prefix, an eight-or-more-digit mixed alphanumeric token, and a known placement suffix. +- [ ] Omit matching slots while preserving evidence/network discovery and emit one deduplicated actionable diagnostic. +- [ ] Add negative tests proving arbitrary stable IDs sharing only the prefix remain eligible. +- [ ] Run the focused tests, then repeat Task 3 verification and delivery. diff --git a/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md index 60140d32e..143794be8 100644 --- a/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md +++ b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md @@ -31,13 +31,20 @@ mistaken for a bot challenge. Cross-page slot inference, merging, and `--replace` otherwise remain unchanged because ambiguous slots never enter those stages. +The `rh-gam-kso__ei_` family is independently known +to be volatile across consecutive crawls. Its render token begins with at least +eight digits and continues with mixed alphanumeric entropy. Discovery refuses +even a single otherwise usable registry or request observation of this narrow +family, preserves the page/network evidence, and emits one site-wide diagnostic. +Arbitrary IDs that merely begin with `rh-gam-kso` do not match this rule. + ## Safety and Output The generator prefers omission over a configuration that cannot match future renders. For the observed Autoblog desktop crawl, replacement output should therefore contain the stable `ad-header-0` and `ad-fixed_bottom-0` slots, while -the in-content collision group is explained in a note. The existing volatile -`rh-gam-kso` group and section-varying sidebar refusal remain unchanged. +the in-content collision group, known `rh-gam-kso` family, and section-varying +sidebar are explained in notes. ## Tests @@ -51,5 +58,8 @@ the in-content collision group is explained in a note. The existing volatile discovery survives when every collided slot is omitted. - A collision-only page is recorded as having evidence rather than as an empty challenge page. +- Single registry- and request-derived `rh-gam-kso` render-token observations + are omitted while retaining evidence and any parseable network ID. +- Stable/nonmatching IDs sharing only the vendor prefix are not omitted. - Existing normalization, request fallback, fragment detection, and full CLI tests remain green. From 02597cf69b569700271d2fc0ae2af09786ddde0c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 15:02:39 +0530 Subject: [PATCH 348/395] Document PR 1013 review remediation --- ...08-19-pr-1013-review-remediation-design.md | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md diff --git a/docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md b/docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md new file mode 100644 index 000000000..1b8567a0b --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md @@ -0,0 +1,98 @@ +# PR #1013 Review Remediation Design + +## Goal + +Resolve every technically sound actionable finding in the two change-request reviews on PR #1013 without broadening the ESI validation spike or changing unrelated runtime behavior. + +## Scope + +The remediation covers the two blocking regressions, the terminal-privacy behavior question, all actionable inline hardening and consistency comments, the experimental-configuration clarification, and archival of the two superseded design documents. + +Three explicitly separate follow-ups remain outside this PR: + +- dependency-version deduplication introduced by the pinned ESI fork; +- gating C2 diagnostic headers before a production rollout; +- upstreaming the ESI parser fixes or publishing a tagged fork release. + +Suggestions that do not fit the current runtime architecture will receive a technical response instead of speculative code. In particular, Fastly constructs `AppState` inside a per-request Wasm instance, so a cache attached to that state does not eliminate cross-request fingerprint work. Fingerprint memoization will be added only if investigation identifies a genuinely longer-lived owner that preserves configuration invalidation. + +## Approach + +Use narrow, test-driven changes grouped by behavioral boundary. Preserve the existing C2 architecture and fail-open/fail-closed contracts. Avoid a general response-policy or template-cache redesign. + +### Cache read and storage safety + +The Fastly cache adapter will consume a found body through fallible `Read::read_to_end`. A stream read error will become the existing cache miss/error path instead of trapping through the SDK's panicking `into_bytes` helper. The declared body length check remains after the read. + +Both transactional and direct inserts will declare `known_length`. The purge-all surrogate key will be exported once from core and consumed by the adapter. Tests will cover a fulfilled reservation not cancelling on drop. + +### Encoding and injection preservation + +`restrict_accept_encoding` will run for every proxied publisher request, including ESI-mode readers that cannot negotiate an assembly encoding. Its existing identity fallback preserves the main-branch guarantee that the response remains processable and receives TSJS injection. + +Regression coverage will exercise malformed or fully refused `Accept-Encoding` input through the relevant request path, not only the helper in isolation. + +### Terminal response privacy + +The response pipeline will distinguish responses that Trusted Server deliberately stamps `private, no-store` from ordinary origin responses that merely arrived with `private` or `no-store` policy. + +A typed response extension will record the former condition at the point where synthesized or assembled HTML is stamped. The Fastly terminal hook will re-enforce `private, no-store` only when that marker is present, after late filter effects. Ordinary proxied responses retain their original cache directives and validators unless another existing policy, such as `Set-Cookie`, independently requires a downgrade. + +Tests will pin both directions: late effects cannot make an assembled response public, and an unrelated origin `private, max-age=600` response keeps its browser-cache semantics and validators. + +### Publisher-content refusal + +C2 authorization will reject documents that already contain the inert TS seam marker rather than rewriting publisher bytes. It will also recognize both case-insensitive ` Date: Wed, 19 Aug 2026 15:04:36 +0530 Subject: [PATCH 349/395] Clarify PR 1013 remediation outcomes --- ...26-08-19-pr-1013-review-remediation-design.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md b/docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md index 1b8567a0b..dcb814dd9 100644 --- a/docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md +++ b/docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md @@ -50,7 +50,7 @@ The directive detector will have one shared implementation used by core authoriz `TemplateCacheKey::to_cache_key` will normalize Vary names to lowercase at the serialization boundary so public fields cannot silently split keys by header-name case. The existing test will pass a literal uppercase name and verify the normalization. -`TemplateMetadata::encode` will reject or prevent carriage-return/newline-bearing values through a typed or fallible boundary consistent with its callers. The implementation will avoid a production panic. Tests will cover injected line breaks and ordinary round trips. +`TemplateMetadata::encode` will become fallible and return a concrete core metadata-encoding error when any encoded scalar or policy-header value contains carriage return or newline. Fastly insert callers will map that error to the existing `TemplateCacheError`; the publisher will then serve the freshly processed private origin response and report `miss-store-error`, matching other cache-write failures. Tests will cover injected line breaks, propagation through the cache adapter, and ordinary round trips. No production assertion or panic will enforce this boundary. The stale schema-prefix assertion will derive its prefix from `TEMPLATE_SCHEMA_VERSION`. The default cache trait behavior will explicitly document its unsupported/null-object role or be made explicit on implementations, whichever is smaller after checking all implementors. @@ -68,15 +68,15 @@ The remediation will: - fold adjacent duplicate implementation blocks together; - remove the redundant test-only `must_use` and add required assertion messages; - move the ESI dependency declaration to workspace dependencies; -- make the example Vary set complete and explain fail-closed coverage; +- make the example Vary set `rsc`, `next-router-state-tree`, `next-router-prefetch`, and `next-router-segment-prefetch`, and explain that every origin `Vary` name must be covered or storage is refused; - correct the request `max-age` documentation and label the mode experimental under #1009; -- remove or simplify the single-variant `PageBidsFormat` residue; +- remove `PageBidsFormat`; accept an absent or `json` format through a direct guard and preserve the existing 400 response for every other value; - remove the dead cached-response content-encoding branch while preserving the identity invariant; - repair non-test rustdoc links; - document the local harness coupling at `build_seam_script`; - list `PlatformTemplateCache` in the platform module roster and consolidate exports; -- reduce CI timing flake risk without weakening the streaming assertion; -- move the two self-labelled historical documents to `docs/superpowers/archive/` and update all references. +- run both C2 workflow invocations with `BID_DELAY=3`, keep the `first_body_byte < complete / 3` assertion, and stop after one diagnostic failure when probe timings are non-numeric instead of performing a second comparison with fabricated zero values; +- move `docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md` and `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` directly into `docs/superpowers/archive/`, then update every reference found by `rg` including links inside the moved documents. ## Error Handling @@ -89,10 +89,10 @@ Each behavioral fix starts with a regression test and runs the narrowest relevan - `cargo fmt --all -- --check`; - all six target-matched clippy aliases; - `cargo test-fastly`, `cargo test-axum`, `cargo test-cloudflare`, and `cargo test-spin`; -- cross-adapter parity and host CLI tests; +- `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity` and `./scripts/test-cli.sh`; - Fastly template-cache and ESI assembly suites; - JavaScript Vitest, build, and formatting; -- documentation formatting and `cargo doc --no-deps --all-features` on an appropriate target; -- the C2 local harness or its CI-equivalent commands when local prerequisites are available. +- `cd docs && npm run format` and the target-matched documentation command selected during planning after checking the workspace target guards; +- `BID_DELAY=3 ./scripts/c2-local-test.sh esi` and `BID_DELAY=3 ./scripts/c2-local-test.sh inline` when Viceroy and the required local artifact are available. Any unavailable local prerequisite will be reported explicitly rather than represented as a passing check. From 8101248531955083d24b8839c07ea0275ae64b41 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 15:13:51 +0530 Subject: [PATCH 350/395] Plan PR 1013 review remediation --- .../2026-08-19-pr-1013-review-remediation.md | 461 ++++++++++++++++++ 1 file changed, 461 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-19-pr-1013-review-remediation.md diff --git a/docs/superpowers/plans/2026-08-19-pr-1013-review-remediation.md b/docs/superpowers/plans/2026-08-19-pr-1013-review-remediation.md new file mode 100644 index 000000000..e44218f87 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-pr-1013-review-remediation.md @@ -0,0 +1,461 @@ +# PR #1013 Review Remediation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve all technically sound actionable review feedback on PR #1013 while preserving ordinary proxy behavior and the ESI spike's fail-safe contracts. + +**Architecture:** Keep the existing publisher, template-cache, and adapter boundaries. Add narrow typed invariants at those boundaries: fallible Fastly body reads, an explicit response-privacy marker, shared publisher-ESI detection, and fallible metadata encoding. Behavioral changes are test-first; mechanical review cleanup follows once runtime contracts are green. + +**Tech Stack:** Rust 2024, `error-stack`, Fastly SDK/Viceroy, `edgezero_core` HTTP types, TypeScript, Vitest, Bash/GitHub Actions. + +**Spec:** `docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md` + +--- + +## File Map + +- `crates/trusted-server-adapter-fastly/src/template_cache.rs`: fallible cache-body reads, insert length metadata, shared purge key, metadata-error propagation. +- `crates/trusted-server-adapter-fastly/src/main.rs`: terminal response effects keyed by an explicit privacy marker. +- `crates/trusted-server-adapter-fastly/src/esi_assembly.rs`: consume the shared publisher-ESI detector and cover comment blocks. +- `crates/trusted-server-core/src/response_privacy.rs`: define and attach the typed terminal-private marker. +- `crates/trusted-server-core/src/platform/template_assembly.rs`: own the shared ESI-directive detector. +- `crates/trusted-server-core/src/platform/template_cache.rs`: shared purge constant, normalized keys, fallible metadata encoding, reservation and schema tests. +- `crates/trusted-server-core/src/platform/mod.rs`: public platform roster/export consistency. +- `crates/trusted-server-core/src/publisher.rs`: unconditional encoding restriction, collision bypass, unused-argument and page-bids cleanup, rustdoc/harness notes. +- `crates/trusted-server-core/src/creative_opportunities.rs` and `src/integrations/gpt_diagnostics.rs`: consolidate adjacent impl blocks and test conventions. +- `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` and `crates/trusted-server-core/src/integrations/gpt_bootstrap.js`: one-shot initial scheduler contract. +- `crates/trusted-server-js/lib/test/integrations/gpt/*.test.ts`: executable scheduling contracts. +- `Cargo.toml`, `crates/trusted-server-adapter-fastly/Cargo.toml`, `.github/workflows/test.yml`, `scripts/c2-local-test.sh`, `trusted-server.example.toml`, and `docs/guide/configuration.md`: dependency, CI, harness, and operator-facing cleanup. +- `docs/superpowers/archive/` plus cross-references: archive the two superseded documents. + +### Task 1: Make Fastly cache I/O fail safely + +**Files:** +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` +- Test: `crates/trusted-server-adapter-fastly/src/template_cache.rs` + +- [ ] **Step 1: Add a failing fallible-reader regression test** + +Extract the byte-reading decision behind a private helper generic over `std::io::Read`, then test it with a reader that returns bytes followed by `io::Error`. The assertion must expect `ReadFoundError::Invalid(TemplateCacheMiss::Truncated)` (or `Backend` if investigation shows the adapter consistently classifies transport failures that way) and must not panic. + +- [ ] **Step 2: Run the focused test and verify RED** + +Run: `cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1 cache_body_read_error` + +Expected: FAIL because the current path uses `Body::into_bytes` and has no fallible helper/classification. + +- [ ] **Step 3: Replace the panicking SDK conversion** + +Import `std::io::Read as _`, call `read_to_end` on `found.to_stream()?`, map the error through `ReadFoundError`, and retain the post-read `metadata.body_len` check. Do not use `into_bytes`. + +- [ ] **Step 4: Add `.known_length(body.len() as u64)` to direct `put`** + +Place it beside `surrogate_keys` and `user_metadata`, matching the reservation insert builder. + +- [ ] **Step 5: Run the adapter cache suite and verify GREEN** + +Run: `cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1 template_cache` + +Expected: all template-cache tests PASS, including the new read-error case. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/template_cache.rs +git commit -m "Make Fastly template cache reads fallible" +``` + +### Task 2: Preserve injection when encoding negotiation fails + +**Files:** +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Add a failing end-to-end publisher test** + +In the ESI publisher tests, send a navigation with `Accept-Encoding: zstd, gzip;q=0, deflate;q=0, br;q=0, identity;q=0`. This fully refuses every representation TS can assemble, so `negotiate_reader_compression` must fail. Queue an origin response that would be undecodable if the header leaked, then assert the recorded origin request advertises `identity` and the returned HTML contains TSJS injection. + +- [ ] **Step 2: Verify RED** + +Run: `cargo test-fastly esi_unsupported_reader_encoding_still_injects_tsjs` + +Expected: FAIL because ESI mode currently skips `restrict_accept_encoding` when `reader_supports_assembly` is false. + +- [ ] **Step 3: Apply the minimal fix** + +Call `restrict_accept_encoding(&mut req)` unconditionally before the origin fetch. Keep reader assembly eligibility separate; it controls shared assembly, not whether the origin offer is processable. + +- [ ] **Step 4: Verify GREEN and the helper matrix** + +Run: `cargo test-fastly publisher_proxy` + +Run: `cargo test-fastly esi_unsupported_reader_encoding_still_injects_tsjs` + +Expected: all matching tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Preserve injection for unsupported encodings" +``` + +### Task 3: Scope terminal privacy re-enforcement to TS-owned responses + +**Files:** +- Modify: `crates/trusted-server-core/src/response_privacy.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Test: `crates/trusted-server-core/src/response_privacy.rs` +- Test: `crates/trusted-server-adapter-fastly/src/main.rs` + +- [ ] **Step 1: Add two failing terminal-policy tests** + +Extend the Fastly tests so an assembled response is explicitly marked and remains `private, no-store` after hostile late effects. Add a companion test whose unmarked origin response starts with `Cache-Control: private, max-age=600`, `ETag`, and `Last-Modified`; after terminal effects it must retain all three. + +- [ ] **Step 2: Verify RED** + +Run: `cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1 terminal_response` + +Expected: the ordinary-origin preservation test FAILS because current code infers TS ownership from the header value. + +- [ ] **Step 3: Add a typed response extension** + +Define a documented marker such as `TerminalPrivateResponse` in `response_privacy.rs`. Make `enforce_synthesized_html_cache_privacy` and the cached-template stamping path insert it when TS creates per-reader output. Keep `enforce_private_no_store` as the pure header mutation used during terminal re-enforcement. + +- [ ] **Step 4: Consume the marker in Fastly terminal effects** + +Replace the `is_private_or_no_store` snapshot with `response.extensions().get::().is_some()`. Apply late effects first, then re-enforce only for marked responses. Leave the existing Set-Cookie privacy guard last. + +- [ ] **Step 5: Verify GREEN across core and Fastly** + +Run: `cargo test-fastly response_privacy` + +Run: `cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1 terminal_response` + +Expected: marked response remains terminal-private; unmarked origin-private response is unchanged. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/response_privacy.rs crates/trusted-server-core/src/publisher.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Scope terminal privacy to synthesized responses" +``` + +### Task 4: Refuse publisher ESI and seam collisions without mutating bytes + +**Files:** +- Modify: `crates/trusted-server-core/src/platform/template_assembly.rs` +- Modify: `crates/trusted-server-core/src/platform/mod.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` +- Test: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` + +- [ ] **Step 1: Rewrite the collision test to express the desired behavior** + +Change `an_origin_marker_collision_is_normalized_before_store` into a regression asserting the cold response preserves the publisher marker bytes, C2 stores no entry, and a second request reaches origin again. Add a script-string collision fixture so the test proves no HTML-comment-only neutralizer is involved. + +- [ ] **Step 2: Add failing ESI-comment tests** + +Test ``, uppercase `", + "", ] { let error = assemble(&template(directive), FRAGMENT) .expect_err("should reject publisher-authored ESI"); diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 7c80f9e12..5bd31b89a 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -57,6 +57,7 @@ pub use image_optimizer::{ pub use kv::UnavailableKvStore; pub use template_assembly::{ PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, + contains_publisher_esi_directive, }; pub use template_cache::REPLAYABLE_POLICY_HEADERS; pub use template_cache::{ diff --git a/crates/trusted-server-core/src/platform/template_assembly.rs b/crates/trusted-server-core/src/platform/template_assembly.rs index 441e7ca31..f56179f1f 100644 --- a/crates/trusted-server-core/src/platform/template_assembly.rs +++ b/crates/trusted-server-core/src/platform/template_assembly.rs @@ -6,6 +6,22 @@ use core::fmt; +/// Whether publisher bytes contain an ESI directive form understood by the parser. +/// +/// Both ordinary `` elements and `` comment blocks are active +/// parser input. The conservative byte scan also rejects these sequences inside scripts: +/// bypassing shared processing is safer than treating publisher data as edge instructions. +#[must_use] +pub fn contains_publisher_esi_directive(bytes: &[u8]) -> bool { + [b"".as_slice(), + b"secret".as_slice(), + b"".as_slice(), + b"".as_slice(), + ] { + assert!( + contains_publisher_esi_directive(directive), + "should detect publisher ESI bytes: {directive:?}" + ); + } + assert!( + !contains_publisher_esi_directive(b""), + "should not classify the inert TS seam as publisher ESI" + ); + } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index f345603c0..4f341cf71 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -61,6 +61,7 @@ use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_eta use crate::integrations::IntegrationRegistry; use crate::platform::{ GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, VarySpec, + contains_publisher_esi_directive, }; use crate::price_bucket::{PriceGranularity, price_bucket}; use crate::response_privacy::enforce_synthesized_html_cache_privacy; @@ -1248,7 +1249,7 @@ fn configured_assembly_mode(settings: &Settings) -> AssemblyMode { .unwrap_or_default() } -/// Whether this mode's `` seam emits [`AD_ASSEMBLY_SEAM`]. +/// Whether this mode's completed template receives [`AD_ASSEMBLY_SEAM`]. /// /// The property that decides whether a template is *expected* to have a hole in it, and /// therefore whether the absence of one is a defect or the design. Only `Esi` splices per @@ -1291,16 +1292,16 @@ fn effective_assembly_mode(settings: &Settings, shared_template_authorized: bool AssemblyMode::Inline } -/// What the `` seam should inject, given the assembly mode. +/// What the streaming HTML processor should inject at ``. /// /// Explicit rather than inferred. The previous shape read /// `ad_slots_script.is_some()` inside the element handler, which silently coupled /// two independent decisions: once [`template_ad_slots_script`] stopped emitting a /// head script under a shared mode, body-close injection stopped with it. /// -/// `Esi` emits [`AD_ASSEMBLY_SEAM`], an inert HTML comment marking where this reader's -/// slots and bids are spliced in. Assembly is a byte split on that comment, performed by -/// this crate on both the miss and the hit path; no ESI layer is involved. +/// `Esi` emits nothing here. Its inert marker is inserted only after the completed +/// transform has been checked for publisher collisions and ESI directives, so TS never +/// has to guess which identical marker belongs to the publisher. pub(crate) fn body_close_injection( mode: AssemblyMode, head_script_present: bool, @@ -1314,9 +1315,7 @@ pub(crate) fn body_close_injection( BodyCloseInjection::None } } - // Constant across every request that reaches the transform — which is what - // makes it safe in a shared template. - AssemblyMode::Esi => BodyCloseInjection::Marker(AD_ASSEMBLY_SEAM.to_string()), + AssemblyMode::Esi => BodyCloseInjection::None, } } @@ -1618,8 +1617,29 @@ pub async fn buffer_publisher_response_async( // request cannot store twice, which would leave nothing for assembly to gate // on. let was_authorized = params.template_cache_key.is_some(); - let bytes = if response_carries_a_seam_marker(was_authorized, settings) { - normalize_fresh_template_seam(bytes) + let contains_publisher_marker = was_authorized + && bytes + .windows(AD_ASSEMBLY_SEAM.len()) + .any(|window| window == AD_ASSEMBLY_SEAM.as_bytes()); + let contains_publisher_esi = was_authorized && contains_publisher_esi_directive(&bytes); + let bypasses_shared_template = contains_publisher_marker || contains_publisher_esi; + if bypasses_shared_template { + log::warn!( + "c2_template_cache bypass: transformed response contains publisher-authored {}", + if contains_publisher_marker { + "seam bytes" + } else { + "ESI" + } + ); + params.template_cache_key.take(); + } + let shared_response_authorized = was_authorized && !bypasses_shared_template; + let bytes = if shared_response_authorized { + insert_before_body_close(bytes, AD_ASSEMBLY_SEAM.as_bytes()) + } else if was_authorized { + let seam = seam_script_for(¶ms); + insert_before_body_close(bytes, seam.as_bytes()) } else { bytes }; @@ -1634,7 +1654,7 @@ pub async fn buffer_publisher_response_async( // The scan runs twice on a miss as a result. A miss is already paying an // origin fetch and a full `lol_html` transform, and a wrong entry in a // shared cache outlives the request that wrote it. - if response_carries_a_seam_marker(was_authorized, settings) { + if response_carries_a_seam_marker(shared_response_authorized, settings) { split_template_at_seam(&bytes).change_context_lazy(|| { crate::error::TrustedServerError::Proxy { message: "refusing to store a template with no usable seam marker" @@ -1642,22 +1662,11 @@ pub async fn buffer_publisher_response_async( } })?; } - // Publisher-authored ESI is outside TS's template schema. If it entered C2, - // a warm hit would stream it without passing through the cold parser that - // rejects it. Revoke the reservation before storage and keep this response - // on the portable byte-seam path. - let contains_publisher_esi = was_authorized && contains_esi_directive(&bytes); - if contains_publisher_esi { - log::warn!( - "c2_template_cache bypass: transformed response contains publisher-authored ESI" - ); - params.template_cache_key.take(); - } let store_outcome = store_template_if_authorized(services, &mut params, &bytes).await; if was_authorized { set_c2_response_state( &mut response, - match (contains_publisher_esi, store_outcome) { + match (bypasses_shared_template, store_outcome) { (true, _) => C2ResponseState::BypassResponse, (false, Some(TemplateStoreOutcome::Stored)) => C2ResponseState::MissStored, (false, Some(TemplateStoreOutcome::Expired)) => { @@ -1669,14 +1678,18 @@ pub async fn buffer_publisher_response_async( }, ); } - let (bytes, assembly_state) = assemble_if_shared( - was_authorized, - !contains_publisher_esi, - settings, - ¶ms, - services, - bytes, - )?; + let (bytes, assembly_state) = if bypasses_shared_template { + (bytes, Some(AssemblyResponseState::ByteSeamFallback)) + } else { + assemble_if_shared( + shared_response_authorized, + shared_response_authorized, + settings, + ¶ms, + services, + bytes, + )? + }; if let Some(state) = assembly_state { set_assembly_response_state(&mut response, state); } @@ -1846,17 +1859,6 @@ fn response_carries_a_seam_marker(was_authorized: bool, settings: &Settings) -> was_authorized && mode_emits_seam_marker(configured_assembly_mode(settings)) } -/// Whether bytes contain an ESI directive that came from publisher content. -/// -/// TS's stored seam is an inert HTML comment, so any ` bool { - bytes - .windows(b"`, or nothing at all. /// /// `seam_ad_slots` is `None` exactly when the ad stack did not run — bot, prefetch, @@ -1931,42 +1933,20 @@ fn split_template_at_seam(template: &[u8]) -> Result<(&[u8], &[u8]), SeamError> Ok((&template[..at], &template[at + marker.len()..])) } -/// Make a freshly transformed ESI template contain one unambiguous seam. +/// Insert a reader payload at the document's final body close, or append it to a fragment. /// -/// `lol_html` emits the marker at ``. HTML fragments and malformed-but-browser- -/// renderable documents may have no body handler, so append a terminal seam rather than -/// converting a valid origin 200 into a TS 500. If any later processing creates an -/// ambiguous result, neutralize every existing marker and mint a new terminal seam. That -/// avoids guessing which occurrence belongs to this transform. -fn normalize_fresh_template_seam(mut template: Vec) -> Vec { - let marker = AD_ASSEMBLY_SEAM.as_bytes(); - let positions = template - .windows(marker.len()) - .enumerate() - .filter_map(|(at, window)| (window == marker).then_some(at)) - .collect::>(); - - if positions.is_empty() { - log::warn!("c2_template_cache transform emitted no body seam; appending a terminal seam"); - template.extend_from_slice(marker); - return template; - } - - if positions.len() > 1 { - let mut escaped = marker.to_vec(); - // Byte 4 is the first byte inside ``; changing it preserves an - // invisible comment and keeps offsets stable. - escaped[4] = b'x'; - for at in &positions { - template[*at..*at + marker.len()].copy_from_slice(&escaped); - } - template.extend_from_slice(marker); - log::warn!( - "c2_template_cache neutralized {} ambiguous seam markers and appended a terminal seam", - positions.len() - ); - } - template +/// The final case-insensitive close matches the streaming pipeline's body-tail convention +/// while avoiding an earlier `""` string in script data. +fn insert_before_body_close(mut document: Vec, payload: &[u8]) -> Vec { + if payload.is_empty() { + return document; + } + let insertion_at = document + .windows(BODY_CLOSE_PREFIX.len()) + .rposition(|window| window.eq_ignore_ascii_case(BODY_CLOSE_PREFIX)) + .unwrap_or(document.len()); + document.splice(insertion_at..insertion_at, payload.iter().copied()); + document } /// Why a template could not be split at its seam. @@ -6980,16 +6960,18 @@ mod tests { #[test] fn shared_template_ad_seam_is_readable_and_versioned() { - let BodyCloseInjection::Marker(marker) = body_close_injection(AssemblyMode::Esi, false) - else { - panic!("ESI mode should emit a shared-template marker"); - }; - assert_eq!( - (crate::platform::TEMPLATE_SCHEMA_VERSION, marker.as_str(),), + (crate::platform::TEMPLATE_SCHEMA_VERSION, AD_ASSEMBLY_SEAM,), (4, ""), "the readable seam and its cache schema must move together" ); + assert!( + matches!( + body_close_injection(AssemblyMode::Esi, false), + BodyCloseInjection::None + ), + "should insert the shared seam only after checking completed publisher bytes" + ); } #[test] @@ -8209,7 +8191,7 @@ mod tests { } #[tokio::test] - async fn publisher_esi_is_never_stored_or_executed() { + async fn publisher_esi_comment_is_never_stored_or_executed() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let assembler = Arc::new(RecordingTemplateAssembler::default()); @@ -8221,8 +8203,7 @@ mod tests { ); stub.push_response_with_headers( 200, - b"publisher" - .to_vec(), + b"".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), ("cache-control", "public, max-age=300"), @@ -8241,7 +8222,11 @@ mod tests { let document = String::from_utf8(body_of(response).await) .expect("served document should be UTF-8"); - assert!(document.to_ascii_lowercase().contains("")); + assert!( + document + .to_ascii_lowercase() + .contains("") + ); assert!(document.contains("scheduleInitialAdInit")); assert!( cache @@ -8499,9 +8484,7 @@ mod tests { let mut request = navigation_request(); request.headers_mut().insert( header::ACCEPT_ENCODING, - HeaderValue::from_static( - "zstd, gzip;q=0, deflate;q=0, br;q=0, identity;q=0", - ), + HeaderValue::from_static("zstd, gzip;q=0, deflate;q=0, br;q=0, identity;q=0"), ); let body = String::from_utf8(body_of(run(&settings, &services, request).await).await) @@ -8634,21 +8617,26 @@ mod tests { } #[test] - fn repeated_fresh_markers_are_all_neutralized_before_a_new_terminal_seam_is_added() { - let input = format!("prefix{AD_ASSEMBLY_SEAM}middle{AD_ASSEMBLY_SEAM}publisher-tail"); + fn seam_payload_is_inserted_before_the_last_body_close() { + let document = br#"

article

"#; - let normalized = normalize_fresh_template_seam(input.into_bytes()); + let inserted = insert_before_body_close(document.to_vec(), b""); assert_eq!( - normalized - .windows(AD_ASSEMBLY_SEAM.len()) - .filter(|window| *window == AD_ASSEMBLY_SEAM.as_bytes()) - .count(), - 1 + inserted, + br#"

article

"#, + "should ignore body-close text inside an earlier script and preserve tag casing" ); - assert!( - normalized.ends_with(AD_ASSEMBLY_SEAM.as_bytes()), - "normalization must mint its own unambiguous terminal seam" + } + + #[test] + fn seam_payload_is_appended_when_the_document_has_no_body_close() { + let inserted = + insert_before_body_close(b"
fragment
".to_vec(), b""); + + assert_eq!( + inserted, b"
fragment
", + "should append the seam payload to an HTML fragment" ); } @@ -8656,7 +8644,9 @@ mod tests { fn parser_validation_does_not_change_the_cached_schema() { assert_eq!(crate::platform::TEMPLATE_SCHEMA_VERSION, 4); assert_eq!(AD_ASSEMBLY_SEAM, ""); - assert!(!contains_esi_directive(AD_ASSEMBLY_SEAM.as_bytes())); + assert!(!contains_publisher_esi_directive( + AD_ASSEMBLY_SEAM.as_bytes() + )); } /// Shareable HTML that already contains the seam marker. @@ -8667,8 +8657,8 @@ mod tests { stub.push_response_with_headers( 200, format!( - "origin{AD_ASSEMBLY_SEAM}\ - {AD_ASSEMBLY_SEAM}" + "

origin

" ) .into_bytes(), vec![ @@ -8679,12 +8669,13 @@ mod tests { } #[tokio::test] - async fn an_origin_marker_collision_is_normalized_before_store() { + async fn origin_marker_collision_bypasses_c2_without_mutating_publisher_bytes() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_html_that_collides_with_the_marker(&stub); + queue_html_that_collides_with_the_marker(&stub); let cold = String::from_utf8( body_of(run(&settings, &services, navigation_request()).await).await, @@ -8699,34 +8690,20 @@ mod tests { cold.contains("origin") && cold.contains("window.tsjs"), "a reserved-comment collision must not turn a valid origin 200 into a 500" ); - assert!(!cold.contains(AD_ASSEMBLY_SEAM)); - assert!(!warm.contains(AD_ASSEMBLY_SEAM)); - assert_eq!(stub.recorded_request_uris().len(), 1); - let entries = cache.entries.lock().expect("should lock entries"); - let template = &entries - .values() - .next() - .expect("template should be stored") - .body; - assert_eq!( - template - .windows(AD_ASSEMBLY_SEAM.len()) - .filter(|window| *window == AD_ASSEMBLY_SEAM.as_bytes()) - .count(), - 1, - "the stored template must retain exactly the transform-owned seam" - ); - let seam_at = template - .windows(AD_ASSEMBLY_SEAM.len()) - .position(|window| window == AD_ASSEMBLY_SEAM.as_bytes()) - .expect("stored template should contain its seam"); - let body_close_at = template - .windows(b"".len()) - .position(|window| window.eq_ignore_ascii_case(b"")) - .expect("fixture should retain its body close"); + for document in [&cold, &warm] { + assert!( + document.contains(&format!("window.publisherMarker=\"{AD_ASSEMBLY_SEAM}\"")), + "should preserve a publisher marker inside script data: {document}" + ); + } + assert_eq!(stub.recorded_request_uris().len(), 2); assert!( - seam_at < body_close_at, - "publisher content after must not be mistaken for TS's seam" + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "should not store a template with a publisher marker collision" ); } From b3ae41facba97cdeab6a38161b486abc4b6ff645 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 15:34:06 +0530 Subject: [PATCH 355/395] Validate and version-guard the SSAT debug comment options Omit the default `[debug.auction_html_comment_options]` table from serialized config blobs so an older binary's `deny_unknown_fields` `DebugConfig` still accepts a pushed blob during a mixed-version deployment or rollback. A non-default table still serializes. Reject `metadata_keys` entries outside the fixed allowlist at config load instead of silently rendering an empty metadata object, keeping the render-time intersection as defense-in-depth. Validate and describe redacted error classifications through the orchestrator's `ERROR_TYPE_*` constants, with a test that fails when a new classification ships without safe wording. Document that `metadata_keys` also gates the three validated keys in upstream mode and is ignored in full mode, and drop it from the full-verbosity recipe where it has no effect. --- .../src/auction/orchestrator.rs | 21 ++- crates/trusted-server-core/src/publisher.rs | 49 ++++-- crates/trusted-server-core/src/settings.rs | 155 +++++++++++++++++- docs/guide/auction-orchestration.md | 34 ++-- trusted-server.example.toml | 2 + 5 files changed, 225 insertions(+), 36 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 1552ee4fd..728cc1efe 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -109,16 +109,29 @@ impl DispatchedAuction { const PROVIDER_ERROR_MESSAGE_CHARS: usize = 500; -const ERROR_TYPE_PARSE_RESPONSE: &str = "parse_response"; -const ERROR_TYPE_LAUNCH_FAILED: &str = "launch_failed"; -const ERROR_TYPE_TRANSPORT: &str = "transport"; -const ERROR_TYPE_TIMEOUT: &str = "timeout"; +pub(crate) const ERROR_TYPE_PARSE_RESPONSE: &str = "parse_response"; +pub(crate) const ERROR_TYPE_LAUNCH_FAILED: &str = "launch_failed"; +pub(crate) const ERROR_TYPE_TRANSPORT: &str = "transport"; +pub(crate) const ERROR_TYPE_TIMEOUT: &str = "timeout"; /// A non-2xx HTTP status from an upstream SSP (e.g. a PBS 4xx/5xx). Distinct /// from [`ERROR_TYPE_TRANSPORT`] (a connection-level failure) so telemetry can /// bucket it separately. `pub(crate)` so producers such as the prebid provider /// tag errors with the exact value the telemetry layer recognises. pub(crate) const ERROR_TYPE_HTTP_STATUS: &str = "http_status"; +/// Every server-owned `error_type` classification. +/// +/// Consumers that reproduce these values — notably the `ts-debug` redaction +/// layer in [`crate::publisher`] — validate against this list so a new +/// classification cannot silently disappear from their output. +pub(crate) const ERROR_TYPE_ALL: &[&str] = &[ + ERROR_TYPE_PARSE_RESPONSE, + ERROR_TYPE_LAUNCH_FAILED, + ERROR_TYPE_TRANSPORT, + ERROR_TYPE_TIMEOUT, + ERROR_TYPE_HTTP_STATUS, +]; + // SECURITY: the returned string is included verbatim (truncated to // PROVIDER_ERROR_MESSAGE_CHARS) in the public /auction response via // ProviderSummary.metadata["message"]. Providers MUST NOT interpolate diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 75b5c0d2d..5dc35f751 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -39,7 +39,9 @@ use crate::auction::endpoints::{ }; use crate::auction::formats::sanitize_publisher_page_url; use crate::auction::orchestrator::{ - AuctionOrchestrator, DispatchAuctionOutcome, DispatchedAuction, + AuctionOrchestrator, DispatchAuctionOutcome, DispatchedAuction, ERROR_TYPE_ALL, + ERROR_TYPE_HTTP_STATUS, ERROR_TYPE_LAUNCH_FAILED, ERROR_TYPE_PARSE_RESPONSE, + ERROR_TYPE_TIMEOUT, ERROR_TYPE_TRANSPORT, }; use crate::auction::telemetry::{ AuctionObservationContext, AuctionSource, AuctionTerminalOutcome, build_auction_events, @@ -1892,15 +1894,14 @@ fn truncate_with_marker(value: &str, max: usize) -> String { } /// Return a recognized server-owned provider error classification. +/// +/// Validates against [`ERROR_TYPE_ALL`] rather than a local literal list so a +/// classification added in the orchestrator cannot drift out of the dump. fn validated_error_type( metadata: &std::collections::HashMap, ) -> Option<&str> { let value = metadata.get("error_type")?.as_str()?; - matches!( - value, - "parse_response" | "launch_failed" | "transport" | "timeout" | "http_status" - ) - .then_some(value) + ERROR_TYPE_ALL.contains(&value).then_some(value) } /// Return a valid HTTP response status from provider metadata. @@ -1914,13 +1915,17 @@ fn validated_http_status( } /// Generate public diagnostic wording without copying provider-controlled text. +/// +/// Every [`ERROR_TYPE_ALL`] entry must map to wording here; the +/// `redacted_metadata_covers_every_orchestrator_error_type` test fails when a +/// new orchestrator classification is added without one. fn safe_error_message(error_type: &str, http_status: Option) -> Option { match error_type { - "parse_response" => Some("Provider response could not be parsed".to_string()), - "launch_failed" => Some("Provider launch failed".to_string()), - "transport" => Some("Provider request failed".to_string()), - "timeout" => Some("Provider request timed out".to_string()), - "http_status" => Some(http_status.map_or_else( + ERROR_TYPE_PARSE_RESPONSE => Some("Provider response could not be parsed".to_string()), + ERROR_TYPE_LAUNCH_FAILED => Some("Provider launch failed".to_string()), + ERROR_TYPE_TRANSPORT => Some("Provider request failed".to_string()), + ERROR_TYPE_TIMEOUT => Some("Provider request timed out".to_string()), + ERROR_TYPE_HTTP_STATUS => Some(http_status.map_or_else( || "Provider returned an HTTP error".to_string(), |status| format!("Provider returned HTTP {status}"), )), @@ -4721,6 +4726,28 @@ mod tests { } } + #[test] + fn redacted_metadata_covers_every_orchestrator_error_type() { + // Drift guard: adding a classification to ERROR_TYPE_ALL without wiring + // wording into safe_error_message would make it vanish from redacted + // dumps through the catch-all match arm. + for error_type in ERROR_TYPE_ALL { + let metadata = std::collections::HashMap::from([( + "error_type".to_string(), + serde_json::json!(error_type), + )]); + assert_eq!( + validated_error_type(&metadata), + Some(*error_type), + "{error_type} should be a recognized classification" + ); + assert!( + safe_error_message(error_type, None).is_some(), + "{error_type} should map to safe diagnostic wording" + ); + } + } + #[test] fn metadata_keys_empty_yields_empty_safe_metadata_in_redacted() { let options = AuctionDebugCommentOptions { diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 27d6f6b8e..745f64023 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1887,7 +1887,16 @@ pub struct DebugConfig { /// Content and verbosity of the `auction_html_comment` dump. Ignored /// when `auction_html_comment` is false. - #[serde(default)] + /// + /// The default table must stay omitted from serialized config blobs: + /// [`DebugConfig`] denies unknown fields, so an older binary rejects a blob + /// carrying this table during a mixed-version deployment or rollback. Any + /// non-default table still serializes and requires restoring a compatible + /// blob before rolling back. + #[serde( + default, + skip_serializing_if = "is_default_auction_debug_comment_options" + )] pub auction_html_comment_options: AuctionDebugCommentOptions, /// Enable the testing-only direct GAM-replace path and the verbose per-bid @@ -1942,6 +1951,11 @@ fn default_auction_debug_metadata_keys() -> Vec { .collect() } +// This predicate preserves rollback compatibility by omitting the default table. +fn is_default_auction_debug_comment_options(value: &AuctionDebugCommentOptions) -> bool { + *value == AuctionDebugCommentOptions::default() +} + /// Behavior of the `` auction dump. Only consulted when /// [`DebugConfig::auction_html_comment`] is true. /// @@ -1949,7 +1963,7 @@ fn default_auction_debug_metadata_keys() -> Vec { /// structs in this file, including the `DebugConfig` this struct nests /// under: an operator typo (e.g. `metadata_key` instead of `metadata_keys`) /// must fail config load loudly, not be silently ignored. -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Serialize)] #[serde(deny_unknown_fields)] pub struct AuctionDebugCommentOptions { /// Include the `provider_responses` section at all. @@ -1965,9 +1979,15 @@ pub struct AuctionDebugCommentOptions { pub include_bids: bool, /// Subset of [`AUCTION_DEBUG_METADATA_ALLOWLIST`] to surface in - /// [`AuctionDebugCommentVerbosity::Redacted`] mode. Keys outside the - /// fixed allowlist are always dropped, config or not. This selector cannot - /// unlock provider diagnostics. Ignored when `verbosity` is `Full`. + /// [`AuctionDebugCommentVerbosity::Redacted`] mode. This selector cannot + /// unlock provider diagnostics, and entries outside the fixed allowlist are + /// rejected at config load by + /// [`validate_metadata_keys`](Self::validate_metadata_keys). + /// + /// [`AuctionDebugCommentVerbosity::Upstream`] builds on the redacted + /// metadata, so this subset still gates those three keys there; the six + /// upstream diagnostics are unlocked by `verbosity` alone. Ignored entirely + /// when `verbosity` is [`AuctionDebugCommentVerbosity::Full`]. #[serde(default = "default_auction_debug_metadata_keys")] pub metadata_keys: Vec, @@ -2012,6 +2032,39 @@ impl AuctionDebugCommentOptions { .filter(|key| !key.is_empty()) .collect(); } + + /// Reject [`Self::metadata_keys`] entries outside + /// [`AUCTION_DEBUG_METADATA_ALLOWLIST`]. + /// + /// Render time intersects the configured list with the allowlist, so an + /// entry outside it is dead config that silently renders `metadata: {}`. + /// Fail the load loudly instead, matching the `deny_unknown_fields` + /// contract on this struct. The render-time intersection stays as + /// defense-in-depth for config paths that bypass this check. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] naming every unknown key. + pub(crate) fn validate_metadata_keys(&self) -> Result<(), Report> { + let unknown: Vec<&str> = self + .metadata_keys + .iter() + .map(String::as_str) + .filter(|key| !AUCTION_DEBUG_METADATA_ALLOWLIST.contains(key)) + .collect(); + + if unknown.is_empty() { + return Ok(()); + } + + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "debug.auction_html_comment_options.metadata_keys contains unsupported keys [{}]; supported keys are [{}]", + unknown.join(", "), + AUCTION_DEBUG_METADATA_ALLOWLIST.join(", ") + ), + })) + } } /// Verbosity of the `ts-debug` auction comment. See @@ -2180,12 +2233,16 @@ impl Settings { /// # Errors /// /// Returns a configuration error if any cached runtime artifact cannot be - /// prepared, if any handler path regex does not compile, or if a creative - /// opportunity slot is invalid. + /// prepared, if any handler path regex does not compile, if a creative + /// opportunity slot is invalid, or if + /// [`AuctionDebugCommentOptions::metadata_keys`] names an unsupported key. pub fn prepare_runtime(&mut self) -> Result<(), Report> { self.image_optimizer.prepare_runtime()?; self.proxy.prepare_runtime()?; self.tinybird.prepare_runtime()?; + self.debug + .auction_html_comment_options + .validate_metadata_keys()?; self.validate_asset_image_optimizer_profile_sets()?; for handler in &self.handlers { @@ -2810,6 +2867,90 @@ mod tests { ); } + #[test] + fn auction_debug_comment_options_unknown_metadata_key_fails_config_load() { + let toml = format!( + "{}\n[debug]\nauction_html_comment = true\n\n[debug.auction_html_comment_options]\nmetadata_keys = [\"http_staus\", \"errors\"]\n", + crate_test_settings_str() + ); + let error = Settings::from_toml(&toml) + .expect_err("should reject metadata keys outside the fixed allowlist"); + let rendered = format!("{error:?}"); + assert!( + rendered.contains("http_staus") && rendered.contains("errors"), + "error should name every unsupported key, got {rendered}" + ); + } + + #[test] + fn auction_debug_comment_options_allowlisted_metadata_keys_load() { + let toml = format!( + "{}\n[debug]\nauction_html_comment = true\n\n[debug.auction_html_comment_options]\nmetadata_keys = [\" message \"]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("should accept an allowlisted key"); + assert_eq!( + settings.debug.auction_html_comment_options.metadata_keys, + vec!["message".to_string()], + "normalize should trim before validation runs" + ); + } + + #[test] + fn auction_debug_comment_options_unknown_field_fails_config_load() { + let result: Result = + toml::from_str(r#"metadata_key = ["message"]"#); + assert!( + result.is_err(), + "a misspelled field must fail config load, not be silently ignored" + ); + } + + #[test] + fn default_auction_debug_comment_options_stay_out_of_serialized_config() { + // Rollback contract: `DebugConfig` denies unknown fields, so the + // previous binary rejects a config blob carrying a table it does not + // know. Defaults must therefore serialize to nothing. + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct LegacyDebugConfig { + #[serde(default)] + ja4_endpoint_enabled: bool, + #[serde(default)] + auction_html_comment: bool, + #[serde(default)] + inject_adm_for_testing: bool, + } + + let value = serde_json::to_value(DebugConfig::default()) + .expect("should serialize the default debug config"); + assert!( + value.get("auction_html_comment_options").is_none(), + "default options table should not be serialized, got {value}" + ); + + let legacy: LegacyDebugConfig = serde_json::from_value(value) + .expect("legacy schema should accept the default debug payload"); + assert!(!legacy.ja4_endpoint_enabled); + assert!(!legacy.auction_html_comment); + assert!(!legacy.inject_adm_for_testing); + + let configured = DebugConfig { + auction_html_comment: true, + auction_html_comment_options: AuctionDebugCommentOptions { + include_bids: false, + ..AuctionDebugCommentOptions::default() + }, + ..DebugConfig::default() + }; + let value = + serde_json::to_value(&configured).expect("should serialize a configured debug config"); + assert!( + value.get("auction_html_comment_options").is_some(), + "non-default options must still serialize, got {value}" + ); + } + #[test] fn tinybird_defaults_to_disabled_placeholders() { let settings = Settings::from_toml(&crate_test_settings_str()) diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index 6fa5cc9f3..a47dbf155 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -839,7 +839,6 @@ auction_html_comment = true include_provider_responses = true include_mediator_response = false include_bids = false -metadata_keys = ["error_type", "http_status", "message"] verbosity = "full" format = "pretty" ``` @@ -855,14 +854,18 @@ responses without spending the dump budget on winning creatives. Raw PBS `debug.httpcalls` and `resolvedrequest` metadata also require `debug = true` under `[integrations.prebid]`. -| Option | Default | Behavior | -| ---------------------------- | -------------------------------------- | ----------------------------------------------------------------------- | -| `include_provider_responses` | `true` | Include the provider response array | -| `include_mediator_response` | `true` | Include the mediator response when a mediator ran | -| `include_bids` | `true` | Include bid objects; when `false`, provider status and metadata remain | -| `metadata_keys` | `error_type`, `http_status`, `message` | Select a subset of the fixed validated metadata keys in `redacted` mode | -| `verbosity` | `redacted` | Select `redacted`, `upstream`, or `full` sensitivity | -| `format` | `compact` | Use compact outer JSON or indented outer JSON with `pretty` | +| Option | Default | Behavior | +| ---------------------------- | -------------------------------------- | ---------------------------------------------------------------------------------------------- | +| `include_provider_responses` | `true` | Include the provider response array | +| `include_mediator_response` | `true` | Include the mediator response when a mediator ran | +| `include_bids` | `true` | Include bid objects; when `false`, provider status and metadata remain | +| `metadata_keys` | `error_type`, `http_status`, `message` | Subset of the fixed validated keys; gates them in `redacted` and `upstream`, ignored in `full` | +| `verbosity` | `redacted` | Select `redacted`, `upstream`, or `full` sensitivity | +| `format` | `compact` | Use compact outer JSON or indented outer JSON with `pretty` | + +`metadata_keys` is a subset selector against a fixed allowlist — +`error_type`, `http_status`, and `message` — never a way to add keys. Any other +entry fails config load rather than being silently ignored. The verbosity modes form an explicit sensitivity ladder: @@ -870,11 +873,14 @@ The verbosity modes form an explicit sensitivity ladder: server-generated `message`, intersected with `metadata_keys`. A successful provider response can therefore have `metadata: {}`. - `upstream` adds provider-controlled errors, warnings, response timings, bid - statuses, and bounded upstream-message fields. It does not include raw PBS - `httpcalls` or `resolvedrequest`. -- `full` includes raw response metadata and untruncated creatives. It can expose - IP addresses, geo data, identifiers, consent strings, request signatures, and - complete provider request/response bodies. + statuses, and bounded upstream-message fields. It builds on the redacted + metadata, so `metadata_keys` still gates the three validated keys, while the + provider diagnostics are unlocked by `verbosity` alone. It does not include + raw PBS `httpcalls` or `resolvedrequest`. +- `full` includes raw response metadata and untruncated creatives, ignoring + `metadata_keys` entirely. It can expose IP addresses, geo data, identifiers, + consent strings, request signatures, and complete provider request/response + bodies. `format = "pretty"` indents only the outer dump. JSON-looking fields such as `requestbody` and `responsebody` remain strings exactly as captured, so their diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 1cbd6f91c..e11329a8d 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -181,6 +181,8 @@ auction_html_comment = false include_provider_responses = true include_mediator_response = true include_bids = true +# Subset of the fixed validated metadata keys shown in "redacted" (and, for +# these three keys, "upstream") mode. Any other key fails config load. metadata_keys = ["error_type", "http_status", "message"] # "redacted" (default), "upstream", or "full". # "upstream" exposes six untyped provider diagnostic values that may contain From 5491204a9eb0a7a044b81754f92f1192abd3cee8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 15:35:21 +0530 Subject: [PATCH 356/395] Harden template cache boundaries --- .../src/template_cache.rs | 44 +++---- .../trusted-server-core/src/platform/mod.rs | 9 +- .../src/platform/template_cache.rs | 119 ++++++++++++++++-- 3 files changed, 135 insertions(+), 37 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs index b3d4fe399..3bc14abfa 100644 --- a/crates/trusted-server-adapter-fastly/src/template_cache.rs +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -20,16 +20,12 @@ use fastly::cache::core::{CacheKey, Found, Transaction}; use std::io::Write as _; use std::time::Duration; use trusted_server_core::platform::{ - PlatformTemplateCache, PlatformTemplateCacheReservation, TemplateCacheError, TemplateCacheKey, + PlatformTemplateCache, PlatformTemplateCacheReservation, + TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, }; -/// Surrogate key attached to every stored template, so a single purge clears them -/// all. This is the rollback lever: without it, backing out a bad template means -/// waiting for the TTL. -const PURGE_ALL_SURROGATE_KEY: &str = "ts-template"; - /// Fastly Core Cache implementation of the C2 template cache. #[derive(Default)] pub struct FastlyTemplateCache; @@ -82,13 +78,11 @@ fn read_found(found: &Found, key: &TemplateCacheKey) -> Result Result<(), TemplateCacheError> { - fastly::http::purge::purge_surrogate_key(PURGE_ALL_SURROGATE_KEY) + fastly::http::purge::purge_surrogate_key(TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY) .map_err(|e| backend_error(format!("purging templates failed: {e:?}"))) } } @@ -332,10 +332,7 @@ mod tests { .expect_err("should reject an abandoned cache stream"); assert!( - matches!( - error, - ReadFoundError::Invalid(TemplateCacheMiss::Truncated) - ), + matches!(error, ReadFoundError::Invalid(TemplateCacheMiss::Truncated)), "should classify a cache stream read failure as truncated" ); } @@ -441,7 +438,12 @@ mod tests { let mut writer = fastly::cache::core::insert(cache_key, Duration::from_secs(0)) .stale_while_revalidate(Duration::from_secs(60)) - .user_metadata(metadata.encode().into()) + .user_metadata( + metadata + .encode() + .expect("valid metadata should encode") + .into(), + ) .execute() .expect("should begin insert"); writer.write_all(&body).expect("should write body"); diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 5bd31b89a..1c5bf4c2a 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -13,6 +13,7 @@ //! - [`PlatformHttpClient`] — outbound HTTP client //! - [`PlatformGeo`] — geographic information lookup //! - [`PlatformTemplateAssembler`] — cold-response shared-template assembly +//! - [`PlatformTemplateCache`] — shared transformed-template caching //! //! ## Platform-Agnostic Components //! @@ -59,11 +60,11 @@ pub use template_assembly::{ PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, contains_publisher_esi_directive, }; -pub use template_cache::REPLAYABLE_POLICY_HEADERS; pub use template_cache::{ - PlatformTemplateCache, PlatformTemplateCacheReservation, TEMPLATE_SCHEMA_VERSION, - TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, - TemplateCacheReservation, TemplateEntry, TemplateMetadata, UnavailableTemplateCache, + PlatformTemplateCache, PlatformTemplateCacheReservation, REPLAYABLE_POLICY_HEADERS, + TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, TEMPLATE_SCHEMA_VERSION, TemplateCacheError, + TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, + TemplateEntry, TemplateMetadata, TemplateMetadataEncodeError, UnavailableTemplateCache, VaryHeaderValues, VarySpec, }; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index ac3761b0f..64e67a907 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -36,6 +36,9 @@ use crate::creative_opportunities::AssemblyMode; /// | 4 | Marker is the shorter, accurate [`AD_ASSEMBLY_SEAM`](crate::publisher::AD_ASSEMBLY_SEAM) | pub const TEMPLATE_SCHEMA_VERSION: u32 = 4; +/// Surrogate key attached to every template so an incident can purge C2 globally. +pub const TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY: &str = "ts-template"; + /// Inputs that select one cached template. /// /// Every field changes the emitted bytes for the same URL. A signal that changes the @@ -105,7 +108,7 @@ impl TemplateCacheKey { &(self.vary_values.len() as u64).to_be_bytes(), ); for varied in &self.vary_values { - push(&mut canonical, varied.name.as_bytes()); + push(&mut canonical, varied.name.to_ascii_lowercase().as_bytes()); match &varied.values { None => push(&mut canonical, b"absent"), Some(values) => { @@ -129,7 +132,10 @@ impl TemplateCacheKey { /// for an incident, the narrow one for ordinary invalidation. #[must_use] pub fn surrogate_keys(&self) -> Vec { - vec!["ts-template".to_string(), self.url_surrogate_key()] + vec![ + TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY.to_string(), + self.url_surrogate_key(), + ] } /// Surrogate key for every variant of this publisher URL. @@ -354,22 +360,51 @@ pub struct TemplateMetadata { pub policy_headers: Vec<(String, String)>, } +/// Why public template metadata could not be represented safely. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +#[display("template metadata field `{field}` contains a line break")] +pub struct TemplateMetadataEncodeError { + field: &'static str, +} + +impl core::error::Error for TemplateMetadataEncodeError {} + impl TemplateMetadata { /// Serialize for `user_metadata`. Deliberately a tiny hand-rolled format rather /// than JSON — one allocation, no dependency, and a parse failure is /// unambiguous. - #[must_use] - pub fn encode(&self) -> Vec { + /// + /// # Errors + /// + /// Returns an error when any public string field contains CR or LF, which would + /// otherwise inject another record into the newline-delimited representation. + pub fn encode(&self) -> Result, TemplateMetadataEncodeError> { + fn reject_line_breaks( + field: &'static str, + value: &str, + ) -> Result<(), TemplateMetadataEncodeError> { + if value.contains(['\r', '\n']) { + return Err(TemplateMetadataEncodeError { field }); + } + Ok(()) + } + + reject_line_breaks("content_encoding", &self.content_encoding)?; + reject_line_breaks("content_type", &self.content_type)?; + for (name, value) in &self.policy_headers { + reject_line_breaks("policy_header_name", name)?; + reject_line_breaks("policy_header_value", value)?; + } + let mut out = format!( "v={}\nce={}\nct={}\nlen={}", self.schema_version, self.content_encoding, self.content_type, self.body_len ); for (name, value) in &self.policy_headers { - // Header values cannot contain newlines (the HTTP parser rejects them), so a - // newline-delimited encoding cannot be broken by a header value. + // Line breaks were rejected above before constructing the delimited form. out.push_str(&format!("\nh={name}:{value}")); } - out.into_bytes() + Ok(out.into_bytes()) } /// Parse `user_metadata`. Returns `None` on anything unexpected, which callers @@ -597,6 +632,11 @@ impl fmt::Debug for dyn PlatformTemplateCache { #[async_trait::async_trait(?Send)] pub trait PlatformTemplateCache: Send + Sync { /// Transactionally look up a template before origin work begins. + /// + /// This compatibility default exists for implementations with no transactional + /// reservation support. It reports ordinary cold misses as `Unsupported`; an + /// adapter that supports C2 reservations must override it so cold requests can + /// return [`TemplateCacheLookup::Reserved`]. async fn lookup_or_reserve( &self, key: &TemplateCacheKey, @@ -731,6 +771,30 @@ mod tests { assert_eq!(cancellations.load(Ordering::SeqCst), 1); } + #[test] + fn fulfilling_a_reservation_does_not_also_cancel_on_drop() { + let cancellations = Arc::new(AtomicUsize::new(0)); + TemplateCacheReservation::new(Box::new(CountingReservation(Arc::clone(&cancellations)))) + .insert( + &TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: Vec::new(), + content_type: "text/html".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 0, + }, + Vec::new(), + std::time::Duration::from_secs(1), + ) + .expect("should fulfil the reservation"); + + assert_eq!( + cancellations.load(Ordering::SeqCst), + 0, + "should discharge a fulfilled reservation without cancelling it" + ); + } + /// Every field must change the key. A field that does not is a cross-serving /// bug: two requests needing different templates would share one entry. #[test] @@ -811,7 +875,10 @@ mod tests { #[test] fn rendered_key_is_fixed_size_and_contains_no_request_material() { let rendered = key().to_cache_key(); - assert_eq!(rendered.len(), "ts-c2-v3-".len() + 64); + assert_eq!( + rendered.len(), + format!("ts-c2-v{TEMPLATE_SCHEMA_VERSION}-").len() + 64 + ); for sensitive in ["example.com", "/news/article", "rsc", "abc123"] { assert!( !rendered.contains(sensitive), @@ -824,7 +891,7 @@ mod tests { fn vary_header_names_are_matched_case_insensitively() { let mut upper = key(); upper.vary_values = vec![VaryHeaderValues { - name: "RSC".to_ascii_lowercase(), + name: "RSC".to_string(), values: Some(vec![b"1".to_vec()]), }]; assert_eq!( @@ -867,7 +934,7 @@ mod tests { fn surrogate_keys_carry_a_global_and_a_per_url_lever() { let keys = key().surrogate_keys(); assert!( - keys.contains(&"ts-template".to_string()), + keys.contains(&TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY.to_string()), "a global purge lever is what makes rollback possible" ); assert_eq!(keys.len(), 2, "global plus per-URL"); @@ -1020,11 +1087,39 @@ mod tests { schema_version: TEMPLATE_SCHEMA_VERSION, body_len: 42, }; - let decoded = - TemplateMetadata::decode(&metadata.encode()).expect("should decode what it encoded"); + let encoded = metadata.encode().expect("valid metadata should encode"); + let decoded = TemplateMetadata::decode(&encoded).expect("should decode what it encoded"); assert_eq!(decoded, metadata); } + #[test] + fn metadata_encoding_rejects_line_break_injection() { + for metadata in [ + TemplateMetadata { + content_encoding: "identity\nh=link:".to_string(), + policy_headers: Vec::new(), + content_type: "text/html".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 0, + }, + TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: vec![( + "content-security-policy".to_string(), + "default-src 'self'\r\nh=link:".to_string(), + )], + content_type: "text/html".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 0, + }, + ] { + assert!( + metadata.encode().is_err(), + "should reject metadata fields that can inject another line" + ); + } + } + #[test] fn unparseable_metadata_is_a_miss_not_a_panic() { for raw in [ From 341bd5237b0381aebcc848071d4e47d59a25e22e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 15:37:37 +0530 Subject: [PATCH 357/395] Make initial GPT scheduling one-shot --- .../src/integrations/gpt_bootstrap.js | 8 +-- .../lib/src/integrations/gpt/index.ts | 31 +++++------ .../integrations/gpt/gpt_bootstrap.test.ts | 16 ++++++ .../gpt/schedule_initial_ad_init.test.ts | 51 +++++++++++++++++++ 4 files changed, 85 insertions(+), 21 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 43934771d..cf0e61a3b 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -94,13 +94,15 @@ // and deliberately identical to the bundle scheduler — the impression is // spent on a viewed tab, and the post-hydration guarantee holds whenever // the request is actually issued. + var initialAdInitScheduled = false; ts.scheduleInitialAdInit = function (initialBids, initialSlots) { - if ((ts.navGeneration || 0) !== 0) return; + if ((ts.navGeneration || 0) !== 0 || initialAdInitScheduled) return; + initialAdInitScheduled = true; // Slots are generation-guarded for the same reason the bids are: the // shared-template seam sends both, and an assignment made before this call // would overwrite a committed SPA navigation's slots. - if (initialSlots) ts.adSlots = initialSlots; - if (initialBids) ts.bids = initialBids; + if (initialSlots !== undefined) ts.adSlots = initialSlots; + if (initialBids !== undefined) ts.bids = initialBids; var fire = function () { if ((ts.navGeneration || 0) !== 0) return; if (typeof ts.adInit === "function") ts.adInit(); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 8c686e75b..a58aa384d 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -667,21 +667,14 @@ function installInitialLoadDetector(ts: TsjsApi): void { * unconditionally at body end would clobber the live bids a faster SPA * navigation already applied. * - * `initialSlots` is passed in for exactly the same reason and was missing it. - * Only the shared-template `` seam sends slots — under `inline` they - * come from the head script, which runs before any navigation can commit — and - * that seam assigned `tsjs.adSlots` on the line *before* calling this. The - * guard protected the bids and `adInit()` while the assignment it was meant to - * protect had already happened, so a committed SPA navigation kept its bids and - * lost its slots. When a navigation has committed since — or - * commits while the deferred callback is pending — the SSR payload is - * dropped and `adInit()` is not run: running anyway would re-run the newer - * route's live slots/bids, destroying and redefining that route's TS slots - * and double-refreshing it. The generation counter (not a URL comparison) - * keeps this guard aligned with the SPA auction hook's own navigation - * identity: a query-only history change the hook ignores must not cancel the - * initial call, while an `/a → /b → /a` round trip — where the URL compares - * equal again — must. + * Shared-template seams pass `initialSlots`; inline documents omit them because + * their head script already installed the slots. An explicit empty array clears + * that state, while omission preserves it. The scheduler accepts only its first + * generation-0 call so duplicate public API calls cannot define and display the + * initial slots twice. If a navigation commits before scheduling or before the + * deferred callback, the SSR payload and `adInit()` are both dropped. The + * generation counter (not a URL comparison) keeps this aligned with the SPA + * auction hook's navigation identity. * * Hidden documents: browsers do not service `requestAnimationFrame` while a * document is hidden, so a background-tab load (Cmd+click, open-in-new-tab) @@ -693,13 +686,15 @@ function installInitialLoadDetector(ts: TsjsApi): void { * holds whenever the request is actually issued. */ function installScheduleInitialAdInit(ts: TsjsApi): void { + let initialAdInitScheduled = false; ts.scheduleInitialAdInit = function ( initialBids?: Record, initialSlots?: AuctionSlot[] ) { - if ((ts.navGeneration ?? 0) !== 0) return; - if (initialSlots) ts.adSlots = initialSlots; - if (initialBids) ts.bids = initialBids; + if ((ts.navGeneration ?? 0) !== 0 || initialAdInitScheduled) return; + initialAdInitScheduled = true; + if (initialSlots !== undefined) ts.adSlots = initialSlots; + if (initialBids !== undefined) ts.bids = initialBids; const runUnlessNavigated = (): void => { if ((ts.navGeneration ?? 0) !== 0) return; ts.adInit?.(); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index 69cb65bfc..8ea28528d 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -119,6 +119,22 @@ describe('gpt_bootstrap.js fallback', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('fallback scheduler accepts only the first schedule call', () => { + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + ts.scheduleInitialAdInit!({ first: { hb_pb: '1.00' } }); + ts.scheduleInitialAdInit!({ second: { hb_pb: '2.00' } }); + expect(ts.bids).toEqual({ first: { hb_pb: '1.00' } }); + + window.dispatchEvent(new Event('load')); + flushFrame(); + flushFrame(); + expect(adInit).toHaveBeenCalledTimes(1); + }); + it('fallback scheduler rides animation frames in a hidden document, holding adInit until first view', () => { // Mirrors the bundle scheduler's intended hidden-tab behavior: rAF is not // serviced while hidden, so a background-tab load holds the initial diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts index 830bac217..81a284596 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts @@ -148,6 +148,22 @@ describe('scheduleInitialAdInit', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('accepts only the first schedule call', async () => { + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + ts.scheduleInitialAdInit!({ first: { hb_pb: '1.00' } }); + ts.scheduleInitialAdInit!({ second: { hb_pb: '2.00' } }); + expect(ts.bids).toEqual({ first: { hb_pb: '1.00' } }); + + window.dispatchEvent(new Event('load')); + flushFrame(); + flushFrame(); + expect(adInit).toHaveBeenCalledTimes(1); + }); + it('still runs after a query-only history change before load', async () => { // The SPA auction hook identifies routes by pathname only, so a query-only // replaceState is not a navigation: it must neither trigger an auction nor @@ -330,6 +346,41 @@ describe('scheduleInitialAdInit', () => { expect(ts.bids).toEqual({ ssr_slot: { hb_pb: '1.00' } }); }); + it('preserves head-injected slots when initialSlots is omitted', async () => { + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + const headSlot = { + id: 'head_slot', + gam_unit_path: '/123/head', + div_id: 'div-head', + formats: [[300, 250]] as Array<[number, number]>, + }; + ts.adSlots = [headSlot]; + + ts.scheduleInitialAdInit!({ head_slot: { hb_pb: '1.00' } }); + + expect(ts.adSlots).toEqual([headSlot]); + }); + + it('replaces existing slots when initialSlots is explicitly empty', async () => { + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + ts.adSlots = [ + { + id: 'stale_slot', + gam_unit_path: '/123/stale', + div_id: 'div-stale', + formats: [[300, 250]], + }, + ]; + + ts.scheduleInitialAdInit!({}, []); + + expect(ts.adSlots).toEqual([]); + }); + it('drops the SSR slot definitions when a navigation has already committed', async () => { // The guard covered the bids and the adInit call, but the shared-template seam // assigned `tsjs.adSlots` on the line *before* calling the scheduler — outside the From eb8e57a43d07968cb8338b066a3a9b8fcd2a36df Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 15:41:31 +0530 Subject: [PATCH 358/395] Remove ESI spike residue --- .../src/creative_opportunities.rs | 3 - .../src/integrations/gpt_diagnostics.rs | 16 +- crates/trusted-server-core/src/publisher.rs | 190 +++++++----------- 3 files changed, 80 insertions(+), 129 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index dc91d6653..890c07cbb 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -372,9 +372,6 @@ impl CreativeOpportunitiesConfig { .unwrap_or(DEFAULT_TEMPLATE_CACHE_MAX_AGE_SECONDS), )) } -} - -impl CreativeOpportunitiesConfig { /// Derives the `{section}` value for `path` under this config's section /// policy ([`section_root`](Self::section_root) and /// [`section_segment`](Self::section_segment)). diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 7a91eead4..487643142 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -112,16 +112,12 @@ impl GptDiagnosticsRequestDecision { ) }) } -} - -impl GptDiagnosticsRequestDecision { /// An active decision, for tests in other modules that need one. /// /// The fields are private and built by `prepare_request` from a cookie or query /// parameter; there is no other way to obtain an active decision across a module /// boundary. #[cfg(test)] - #[must_use] pub(crate) fn active_for_tests() -> Self { Self { active: true, @@ -182,8 +178,16 @@ mod head_seam_invariant_tests { #[test] fn a_default_decision_injects_nothing() { let decision = GptDiagnosticsRequestDecision::default(); - assert_eq!(decision.bootstrap_script(), None); - assert_eq!(decision.module_script_tag(), None); + assert_eq!( + decision.bootstrap_script(), + None, + "should not inject a bootstrap for an inert decision" + ); + assert_eq!( + decision.module_script_tag(), + None, + "should not inject a module for an inert decision" + ); assert!( !decision.requires_private_no_store(), "an inert decision should not force the response private" diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4f341cf71..9b1787ab1 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1467,7 +1467,7 @@ pub(crate) fn classify_response_route( pub struct OwnedProcessResponseParams { /// Where to store the transformed template, or [`None`] to store nothing. /// - /// `Some` only when [`c2_bypass_reason`] cleared the response, so the key's + /// `Some` only when `c2_bypass_reason` cleared the response, so the key's /// presence *is* the decision — there is no second place that could disagree with /// the gate, and no way to reach the store without having passed it. /// @@ -1662,7 +1662,7 @@ pub async fn buffer_publisher_response_async( } })?; } - let store_outcome = store_template_if_authorized(services, &mut params, &bytes).await; + let store_outcome = store_template_if_authorized(&mut params, &bytes).await; if was_authorized { set_c2_response_state( &mut response, @@ -1997,16 +1997,8 @@ fn build_cached_template_response( HeaderValue::from_str(&entry.metadata.content_type) .change_context_lazy(|| invalid("content type"))?, ); - // Stored templates are identity bytes; final reader negotiation is applied after - // assembly rather than replaying the origin's representation. - if !entry.metadata.content_encoding.is_empty() && entry.metadata.content_encoding != "identity" - { - response.headers_mut().insert( - header::CONTENT_ENCODING, - HeaderValue::from_str(&entry.metadata.content_encoding) - .change_context_lazy(|| invalid("content encoding"))?, - ); - } + // Metadata decoding accepts only identity templates. Reader encoding is selected + // after assembly rather than replaying the origin's representation. // No `Content-Length`. The assembled length is not known until bids resolve, and on // this adapter headers commit before the first body byte — so a length guessed here // could not be corrected later. @@ -2037,7 +2029,7 @@ fn build_cached_template_response( /// Writes the transformed template to the shared cache, if the gate authorized it. /// /// The key's presence is the authorization: it is `Some` only when -/// [`c2_bypass_reason`] cleared the response, so this cannot store something the gate +/// `c2_bypass_reason` cleared the response, so this cannot store something the gate /// rejected. Takes the key rather than borrowing it, so a second call for the same /// request stores nothing. /// @@ -2047,7 +2039,6 @@ fn build_cached_template_response( /// /// Spike-only, for the #1009 ESI validation. async fn store_template_if_authorized( - _services: &RuntimeServices, params: &mut OwnedProcessResponseParams, bytes: &[u8], ) -> Option { @@ -5086,6 +5077,9 @@ pub(crate) fn build_seam_script( slots_json: &str, bid_map: &serde_json::Map, ) -> String { + // `scripts/c2-local-test.sh` probes the minified `var a=JSON.parse`, + // `var b=JSON.parse`, and `s(b,a)` literals below. Update the harness with any + // semantically equivalent rewrite so its black-box checks keep matching output. let bids = serde_json::to_string(bid_map) .expect("serde_json::to_string of Map should be infallible"); format!( @@ -5937,30 +5931,6 @@ fn page_bids_unknown_format() -> Response { /// The SPA hook sends `location.pathname`, but the parameter is /// client-controlled: strip any query string or fragment and force a leading /// `/` so slot `page_patterns` always match against a canonical path shape. -/// How the page-bids endpoint serializes its answer. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub(crate) enum PageBidsFormat { - /// `application/json`. What the SPA navigation hook consumes. - #[default] - Json, -} - -impl PageBidsFormat { - /// Parse the `format` query parameter. - /// - /// # Errors - /// - /// Returns the offending value if it names no known format. Unknown values are - /// rejected rather than defaulting so callers cannot silently negotiate a response - /// representation the endpoint no longer supports. - fn parse(raw: Option<&str>) -> Result { - match raw { - None | Some("json") => Ok(Self::Json), - Some(other) => Err(other.to_string()), - } - } -} - fn normalize_page_bids_path(raw: &str) -> String { let path = raw.split(['?', '#']).next().unwrap_or(""); if path.starts_with('/') { @@ -6070,22 +6040,18 @@ pub async fn handle_page_bids( }) .unwrap_or_else(|| "/".to_string()); - let format = match PageBidsFormat::parse( - req.uri() - .query() - .and_then(|query| { - url::form_urlencoded::parse(query.as_bytes()) - .find(|(k, _)| k == "format") - .map(|(_, v)| v.into_owned()) - }) - .as_deref(), - ) { - Ok(format) => format, - Err(unknown) => { - log::warn!("page-bids: rejecting unknown format `{unknown}`"); - return Ok(page_bids_unknown_format()); - } - }; + let format = req.uri().query().and_then(|query| { + url::form_urlencoded::parse(query.as_bytes()) + .find(|(key, _)| key == "format") + .map(|(_, value)| value.into_owned()) + }); + if !matches!(format.as_deref(), None | Some("json")) { + log::warn!( + "page-bids: rejecting unknown format `{}`", + format.as_deref().unwrap_or_default() + ); + return Ok(page_bids_unknown_format()); + } let matched_slots = match_renderable_slots(auction.slots, co_config, &path_param); @@ -6279,7 +6245,6 @@ pub async fn handle_page_bids( Vec::new() }; - debug_assert_eq!(format, PageBidsFormat::Json); let body = serde_json::json!({ "slots": slots_json, "bids": bid_map, @@ -7213,32 +7178,6 @@ mod tests { use super::*; - #[test] - fn an_absent_format_is_json_so_existing_clients_are_unaffected() { - assert_eq!(PageBidsFormat::parse(None), Ok(PageBidsFormat::Json)); - assert_eq!( - PageBidsFormat::parse(Some("json")), - Ok(PageBidsFormat::Json) - ); - } - - #[test] - fn the_removed_fragment_format_is_rejected() { - assert_eq!( - PageBidsFormat::parse(Some("fragment")), - Err("fragment".to_string()) - ); - } - - #[test] - fn an_unknown_format_is_rejected_rather_than_defaulting_to_json() { - assert_eq!( - PageBidsFormat::parse(Some("scrpit")), - Err("scrpit".to_string()) - ); - assert_eq!(PageBidsFormat::parse(Some("")), Err(String::new())); - } - #[test] fn an_unknown_format_response_is_not_storable() { // Every response from this endpoint is per-user. An error is no exception: @@ -7261,10 +7200,6 @@ mod tests { //! storing twice for one request. use super::*; - use crate::platform::ClientInfo; - use crate::platform::test_support::{ - NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, - }; /// Records what was stored, so the assertions are about behaviour rather than /// about a call not returning an error. @@ -7367,19 +7302,6 @@ mod tests { } } - fn services_with(cache: Arc) -> RuntimeServices { - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .geo(Arc::new(NoopGeo)) - .http_client(Arc::new(StubHttpClient::new())) - .client_info(ClientInfo::default()) - .template_cache(cache) - .build() - } - #[tokio::test] async fn an_unauthorized_response_stores_nothing() { // `None` is what the gate leaves behind on every bypass, and it is also the @@ -7390,12 +7312,7 @@ mod tests { let mut params = make_stream_params(&settings, "identity"); params.template_cache_key = None; - let _ = store_template_if_authorized( - &services_with(Arc::clone(&cache)), - &mut params, - b"body", - ) - .await; + let _ = store_template_if_authorized(&mut params, b"body").await; assert!( cache.recorded().is_empty(), @@ -7410,12 +7327,7 @@ mod tests { let mut params = make_stream_params(&settings, "identity"); params.template_cache_key = Some(authorization(&cache)); - let _ = store_template_if_authorized( - &services_with(Arc::clone(&cache)), - &mut params, - b"transformed", - ) - .await; + let _ = store_template_if_authorized(&mut params, b"transformed").await; assert_eq!( cache.recorded(), @@ -7433,10 +7345,8 @@ mod tests { let settings = create_test_settings(); let mut params = make_stream_params(&settings, "identity"); params.template_cache_key = Some(authorization(&cache)); - let services = services_with(Arc::clone(&cache)); - - let _ = store_template_if_authorized(&services, &mut params, b"first").await; - let _ = store_template_if_authorized(&services, &mut params, b"second").await; + let _ = store_template_if_authorized(&mut params, b"first").await; + let _ = store_template_if_authorized(&mut params, b"second").await; assert_eq!( cache.recorded().len(), @@ -7454,12 +7364,7 @@ mod tests { expired.expires_at = Instant::now(); params.template_cache_key = Some(expired); - let outcome = store_template_if_authorized( - &services_with(Arc::clone(&cache)), - &mut params, - b"body", - ) - .await; + let outcome = store_template_if_authorized(&mut params, b"body").await; assert_eq!(outcome, Some(TemplateStoreOutcome::Expired)); assert!( @@ -17141,6 +17046,51 @@ mod tests { make_page_bids_request_on(PAGE_BIDS_PATH, path) } + #[tokio::test] + async fn page_bids_format_absent_or_json_returns_json() { + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + for path_and_format in ["/2024/article", "/2024/article&format=json"] { + let response = run_page_bids_response( + &settings, + &orchestrator, + &[], + make_page_bids_request(path_and_format), + ) + .await; + assert_eq!( + response.status(), + StatusCode::OK, + "should accept page-bids format in `{path_and_format}`" + ); + assert_eq!( + response.headers().get(header::CONTENT_TYPE), + Some(&HeaderValue::from_static("application/json")), + "should return JSON for `{path_and_format}`" + ); + } + } + + #[tokio::test] + async fn page_bids_format_rejects_removed_unknown_and_empty_values() { + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + for format in ["fragment", "scrpit", ""] { + let response = run_page_bids_response( + &settings, + &orchestrator, + &[], + make_page_bids_request(&format!("/2024/article&format={format}")), + ) + .await; + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "should reject page-bids format `{format}`" + ); + } + } + /// Builds a page-bids request against an explicit endpoint path, so the /// canonical route and its deprecated alias can be compared directly. fn make_page_bids_request_on(endpoint: &str, path: &str) -> Request { From d0c3cb759c9b7d632e77dca8eef279d579a520f2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 15:44:57 +0530 Subject: [PATCH 359/395] Align ESI spike documentation and tooling --- .github/workflows/test.yml | 4 +-- Cargo.toml | 1 + .../trusted-server-adapter-fastly/Cargo.toml | 2 +- .../trusted-server-core/src/html_processor.rs | 2 +- .../src/platform/template_cache.rs | 2 +- crates/trusted-server-core/src/publisher.rs | 4 +-- docs/guide/configuration.md | 15 ++++++++-- ...08-esi-cacheable-root-validation-design.md | 24 +++++++-------- .../2026-08-10-1009-esi-validation-spike.md | 8 ++--- ...2026-08-08-1009-measurement-and-stage-0.md | 6 ++-- .../2026-08-12-1009-esi-merge-hardening.md | 2 +- .../2026-08-19-pr-1013-review-remediation.md | 17 ++++++++--- ...11-1009-streaming-assembly-architecture.md | 2 +- ...08-19-pr-1013-review-remediation-design.md | 2 +- scripts/c2-local-test.sh | 29 ++++++++++--------- trusted-server.example.toml | 10 +++++-- 16 files changed, 79 insertions(+), 51 deletions(-) rename docs/superpowers/{specs => archive}/2026-08-08-esi-cacheable-root-validation-design.md (97%) rename docs/superpowers/{plans => archive}/2026-08-10-1009-esi-validation-spike.md (99%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f0bb6cfb..6f9262a14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,10 +59,10 @@ jobs: run: cargo test-fastly - name: Run C2 ESI local harness - run: ./scripts/c2-local-test.sh esi + run: BID_DELAY=3 ./scripts/c2-local-test.sh esi - name: Run inline control harness - run: ./scripts/c2-local-test.sh inline + run: BID_DELAY=3 ./scripts/c2-local-test.sh inline test-axum: name: cargo test (axum native) diff --git a/Cargo.toml b/Cargo.toml index 754a241fd..cbb143564 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ edgezero-cli = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4" } edgezero-core = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } env_logger = "0.11" error-stack = "0.6" +esi = { git = "https://github.com/stackpop/esi.git", rev = "4c53feab4d22ad9a84641b4c46f3f63bc6d197e2" } fastly = "0.12" fern = "0.7.1" flate2 = "1.1" diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 3d42ae388..47cc609b2 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -19,7 +19,7 @@ derive_more = { workspace = true } edgezero-adapter-fastly = { workspace = true, features = ["fastly"] } edgezero-core = { workspace = true } error-stack = { workspace = true } -esi = { git = "https://github.com/stackpop/esi.git", rev = "4c53feab4d22ad9a84641b4c46f3f63bc6d197e2" } +esi = { workspace = true } fastly = { workspace = true } fern = { workspace = true } futures = { workspace = true } diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 711c4e31d..083ca1a6b 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -163,7 +163,7 @@ impl StreamProcessor for HtmlWithPostProcessing { /// which coupled two independent choices: once a shared-template mode stopped /// emitting the head script, body-close injection silently stopped too. /// -/// See `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +/// See `docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md` /// §6.7. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub enum BodyCloseInjection { diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index 64e67a907..35823c1c4 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -11,7 +11,7 @@ //! //! C2 holds a *shared template*: no per-user bytes, and no decisions that depend on //! the request. What may and may not live in it is -//! [§6.7 of the design doc](../../../../docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md), +//! [§6.7 of the design doc](../../../../docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md), //! and the invariant is enforced by the rendered-document byte-identity tests in //! `publisher`. //! diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 9b1787ab1..0cba0bba3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -5386,7 +5386,7 @@ impl C2CachePolicy { /// Leak vectors are checked before mere ineligibility so the reported reason is /// the most serious one that applies. /// -/// See `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +/// See `docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md` /// §6.6 for why C1, C2 and a final assembled-response cache are distinct, and why /// the third must never exist. #[cfg(test)] @@ -5748,7 +5748,7 @@ fn c2_cache_ttl( /// So ESI returns [`None`] **unconditionally**, and `adSlots` moves to the /// per-request body seam alongside the bids. The head is not a template hole. /// -/// See `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +/// See `docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md` /// §6.7. pub(crate) fn template_ad_slots_script( mode: AssemblyMode, diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 531608f8f..36bd8af0d 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1369,6 +1369,10 @@ formats = [{ width = 728, height = 90 }] ### Shared template assembly (`assembly_mode = "esi"`) +This configuration is an experimental validation spike scoped to +[IABTechLab/trusted-server#1009](https://github.com/IABTechLab/trusted-server/issues/1009), +not a settled production cache interface. + `assembly_mode` controls how initial-page slot and bid state is delivered: - `inline` (default) transforms every origin response and injects the current @@ -1403,7 +1407,12 @@ assembly_mode = "esi" # Every request header, except Accept-Encoding, that the publisher origin can # name in Vary for these documents. Names are validated and de-duplicated. -template_cache_vary = ["rsc", "next-router-prefetch"] +template_cache_vary = [ + "rsc", + "next-router-state-tree", + "next-router-prefetch", + "next-router-segment-prefetch", +] # Safety ceiling for the shared template. Defaults to 60; valid range 1–86400. # The origin's remaining edge freshness may make the actual lifetime shorter. @@ -1433,7 +1442,7 @@ lifetime is capped by `template_cache_max_age_seconds`. A browser reload commonly sends `Cache-Control: max-age=0`. TS may reuse a fresh reader-neutral C2 template for that reload, but it still builds a new private response and runs a new per-reader auction. Explicit `no-cache`, `no-store`, -positive or malformed `max-age`, range, and conditional requests still bypass C2. +positive or malformed request `max-age`, range, and conditional requests still bypass C2. Check `X-TS-C2-Cache: hit` to verify template reuse. `template_cache_vary` is necessary because lookup occurs before the origin can @@ -1449,6 +1458,8 @@ document's meaning based on `Accept-Encoding`. Never put `Cookie` in reader-neutral template. With `origin_is_cookie_independent = false` (the safe default), all cookie-bearing requests bypass. With it set to `true`, an origin `Vary: Cookie` still overrides the assertion and refuses storage. +Every other name the origin emits in `Vary` must appear in the configured list; +an uncovered name safely refuses template storage. For a canary, inspect `X-TS-C2-Cache`. Its bounded values are `hit`, `miss-stored`, `miss-store-error`, `miss-reserved`, `bypass-request`, diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md similarity index 97% rename from docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md rename to docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md index 031ecc50c..bec425f82 100644 --- a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md +++ b/docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md @@ -43,7 +43,7 @@ cross-reference point at it. The subject moved, the path did not._ > other direction. > > **ESI is therefore feasible and unvalidated, not rejected.** Validating it is -> [a separate plan](../plans/2026-08-10-1009-esi-validation-spike.md). What survives +> [a separate plan](./2026-08-10-1009-esi-validation-spike.md). What survives > here is the latency re-diagnosis and the Stage 0 optimisation, which are worth doing > and are **not** an answer to #1009. @@ -52,12 +52,12 @@ cross-reference point at it. The subject moved, the path did not._ #1009 is answered across three documents, not one. This is the only place that says which owns what. -| Document | Owns | -| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------- | -| **This spec** | Why the TTFB regression happens, what Stage 0 is and why, and the corrected ESI feasibility verdict | -| [Stage 0 plan](../plans/2026-08-08-1009-measurement-and-stage-0.md) | Implementing the measurement and the cache-bypass flag. **Does not close #1009.** | -| [ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md) | **Where #1009 is actually decided.** Four arms, safety gates, decision rule. | -| [Findings](../plans/2026-08-08-1009-measurement-findings.md) | Recorded results. Currently: Step A only, at `PROVISIONAL PASS`. | +| Document | Owns | +| ------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| **This spec** | Why the TTFB regression happens, what Stage 0 is and why, and the corrected ESI feasibility verdict | +| [Stage 0 plan](../plans/2026-08-08-1009-measurement-and-stage-0.md) | Implementing the measurement and the cache-bypass flag. **Does not close #1009.** | +| [ESI validation spike](./2026-08-10-1009-esi-validation-spike.md) | **Where #1009 is actually decided.** Four arms, safety gates, decision rule. | +| [Findings](../plans/2026-08-08-1009-measurement-findings.md) | Recorded results. Currently: Step A only, at `PROVISIONAL PASS`. | **If you want the ESI answer**, it is [§2](#2-why--the-three-findings) for the verdict, [§6.6](#66-the-esi-pipeline-corrected) for the pipeline, and the spike plan for how it @@ -91,7 +91,7 @@ gets validated. Everything else here is Stage 0 and the latency analysis behind | # | Decision | Owner needed | | --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | -| D1 | **ESI is feasible and unvalidated.** Validate it via [the spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), not by deferring it. | Eng + product | +| D1 | **ESI is feasible and unvalidated.** Validate it via [the spike plan](./2026-08-10-1009-esi-validation-spike.md), not by deferring it. | Eng + product | | D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | | D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to a **`FINAL PASS`** in §3. A `Vary` check alone is a `PROVISIONAL PASS` and is not a release gate. Rollback needs a purge path, not only a config push. | Eng | | D4 | **Stages 1–2 queue behind the SSAT price defect and #938.** Stages 3b–4 unscheduled. ESI is not in this queue — see §7. | Product | @@ -110,7 +110,7 @@ include tags into a shared template; `fastly::cache::core` stores that template; already a dependency. The real open questions are empirical, not architectural: does it beat a plain client fetch by enough to justify a Fastly-only rendering path, and can per-user leakage be excluded under cold MISS, warm HIT, stale revalidation, and fragment -failure. [The spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) answers +failure. [The spike plan](./2026-08-10-1009-esi-validation-spike.md) answers those; [§6.6](#66-the-esi-pipeline-corrected) gives the pipeline. Two constraints stay true regardless. ESI is **Fastly-only at every API level**, so it @@ -522,7 +522,7 @@ differing in consent, bot classification, and prefetch status** — not an absen per-user-values scan, which the broken design would have passed. This applies to any shared-template work, ESI or client-fill alike. The -[spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) implements it. +[spike plan](./2026-08-10-1009-esi-validation-spike.md) implements it. --- @@ -542,7 +542,7 @@ rather than repeated here; everything below it is deferred. **ESI is not a stage here.** It was Stage 5 in an earlier revision, queued behind the rest. It no longer queues: it is feasible on the pinned SDK and is decided by -[its own spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), which runs +[its own spike plan](./2026-08-10-1009-esi-validation-spike.md), which runs independently of Stages 1–4. The shared template cache it needs is `fastly::cache::core` ([§6.6](#66-the-esi-pipeline-corrected)), not a new service. @@ -785,7 +785,7 @@ exists. ## Appendix E — ESI notes (condensed) -Input to [the ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md). +Input to [the ESI validation spike](./2026-08-10-1009-esi-validation-spike.md). Recording only what would otherwise be re-derived: - Pin `esi = "0.7"`. Pre-1.0, irregular cadence, two yanked betas in the 0.7 line. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/archive/2026-08-10-1009-esi-validation-spike.md similarity index 99% rename from docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md rename to docs/superpowers/archive/2026-08-10-1009-esi-validation-spike.md index 5b9a455e7..1d83665ca 100644 --- a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md +++ b/docs/superpowers/archive/2026-08-10-1009-esi-validation-spike.md @@ -31,9 +31,9 @@ request-dependent decisions. Assembly is either the `esi` crate (edge) or a clie **Tech Stack:** Rust 2024, `wasm32-wasip1`, `fastly` 0.12.1 (`cache::core`, `http::purge`), `esi` 0.7, `lol_html`, a real Fastly test service for cache behaviour. -**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` — +**Spec:** `docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md` — read the 2026-08-10 correction at the top and -[§6.6](../specs/2026-08-08-esi-cacheable-root-validation-design.md#66-the-esi-pipeline-corrected) +[§6.6](./2026-08-08-esi-cacheable-root-validation-design.md#66-the-esi-pipeline-corrected) before writing any code. **Control:** [the Stage 0 plan](./2026-08-08-1009-measurement-and-stage-0.md). Its @@ -671,9 +671,9 @@ deployable — a cache that works is necessary, not sufficient. ## Task 4: Arm A2 — client-fill Mostly already specified. See -[the spec's Appendix B](../specs/2026-08-08-esi-cacheable-root-validation-design.md#appendix-b--stage-1-plumbing-condensed) +[the spec's Appendix B](./2026-08-08-esi-cacheable-root-validation-design.md#appendix-b--stage-1-plumbing-condensed) for the client plumbing, the two-condition join gate, and the server contract; and -[§5](../specs/2026-08-08-esi-cacheable-root-validation-design.md#5-the-trap-in-the-deferred-work--read-this-before-scheduling-stages-12) +[§5](./2026-08-08-esi-cacheable-root-validation-design.md#5-the-trap-in-the-deferred-work--read-this-before-scheduling-stages-12) for the silent-empty-bids trap, which applies in full. - [ ] **Step 1: Hoist the closure-trapped client state** — `pageBidsEndpoint`, diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md index 6de35c9b3..eafb5b0a4 100644 --- a/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md @@ -9,7 +9,7 @@ later work is compared against. > **This plan does not close #1009.** It contains no ESI arm and no client-fill arm, so > completing it cannot answer whether ESI separates cacheable content from per-user > state. It is a **supporting optimisation and the experimental control** for -> [the ESI validation spike](./2026-08-10-1009-esi-validation-spike.md), which is where +> [the ESI validation spike](../archive/2026-08-10-1009-esi-validation-spike.md), which is where > #1009 is actually decided. Scoped and framed this way after external review on > 2026-08-10. @@ -22,7 +22,7 @@ Stages 1–2 in the spec and are explicitly out of scope. **Tech Stack:** Rust 2024 edition, `wasm32-wasip1`, Fastly Compute, `web_time::Instant` for wasm-safe timing, `log` for instrumentation, Viceroy for adapter tests. -**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +**Spec:** `docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md` (§3 Steps A/B/C and §4 Stage 0). Read §4 and §5 before starting Task 5. **Before pushing, run both documentation gates:** @@ -551,7 +551,7 @@ config flip rather than a second deploy — see Task 5. Replace the block at ```rust // Single source of truth for the request and the log line below. Operator- // controlled so the read-through cache can be re-enabled without a release; -// see docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md §4. +// see docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md §4. let cache_bypass = should_run_ad_stack && settings.publisher.bypass_origin_cache; if cache_bypass { platform_request = platform_request.with_cache_bypass(); diff --git a/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md index a2b388594..6cf632424 100644 --- a/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md +++ b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md @@ -199,7 +199,7 @@ TypeScript/Vitest, Viceroy, shell harness. - Modify: `trusted-server.example.toml` - Modify: `docs/guide/configuration.md` -- Modify: `docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md` +- Modify: `docs/superpowers/archive/2026-08-10-1009-esi-validation-spike.md` - Modify: `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md` - Modify: relevant #1009 findings documents diff --git a/docs/superpowers/plans/2026-08-19-pr-1013-review-remediation.md b/docs/superpowers/plans/2026-08-19-pr-1013-review-remediation.md index e44218f87..f4395bccc 100644 --- a/docs/superpowers/plans/2026-08-19-pr-1013-review-remediation.md +++ b/docs/superpowers/plans/2026-08-19-pr-1013-review-remediation.md @@ -31,6 +31,7 @@ ### Task 1: Make Fastly cache I/O fail safely **Files:** + - Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` - Test: `crates/trusted-server-adapter-fastly/src/template_cache.rs` @@ -68,6 +69,7 @@ git commit -m "Make Fastly template cache reads fallible" ### Task 2: Preserve injection when encoding negotiation fails **Files:** + - Modify: `crates/trusted-server-core/src/publisher.rs` - Test: `crates/trusted-server-core/src/publisher.rs` @@ -103,6 +105,7 @@ git commit -m "Preserve injection for unsupported encodings" ### Task 3: Scope terminal privacy re-enforcement to TS-owned responses **Files:** + - Modify: `crates/trusted-server-core/src/response_privacy.rs` - Modify: `crates/trusted-server-core/src/publisher.rs` - Modify: `crates/trusted-server-adapter-fastly/src/main.rs` @@ -145,6 +148,7 @@ git commit -m "Scope terminal privacy to synthesized responses" ### Task 4: Refuse publisher ESI and seam collisions without mutating bytes **Files:** + - Modify: `crates/trusted-server-core/src/platform/template_assembly.rs` - Modify: `crates/trusted-server-core/src/platform/mod.rs` - Modify: `crates/trusted-server-core/src/publisher.rs` @@ -196,6 +200,7 @@ git commit -m "Refuse publisher ESI and seam collisions" ### Task 5: Harden cache keys, metadata, and reservations **Files:** + - Modify: `crates/trusted-server-core/src/platform/template_cache.rs` - Modify: `crates/trusted-server-core/src/platform/mod.rs` - Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` @@ -241,6 +246,7 @@ git commit -m "Harden template cache boundaries" ### Task 6: Make initial GPT scheduling one-shot **Files:** + - Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` - Modify: `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` - Modify: `crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts` @@ -282,6 +288,7 @@ git commit -m "Make initial GPT scheduling one-shot" ### Task 7: Remove spike residue and apply Rust consistency fixes **Files:** + - Modify: `crates/trusted-server-core/src/publisher.rs` - Modify: `crates/trusted-server-core/src/creative_opportunities.rs` - Modify: `crates/trusted-server-core/src/integrations/gpt_diagnostics.rs` @@ -320,14 +327,14 @@ git commit -m "Remove ESI spike residue" ### Task 8: Align manifests, examples, CI, and historical docs **Files:** + - Modify: `Cargo.toml` - Modify: `crates/trusted-server-adapter-fastly/Cargo.toml` - Modify: `.github/workflows/test.yml` - Modify: `scripts/c2-local-test.sh` - Modify: `trusted-server.example.toml` - Modify: `docs/guide/configuration.md` -- Move: `docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md` to `docs/superpowers/archive/2026-08-10-1009-esi-validation-spike.md` -- Move: `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` to `docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md` +- Move the #1009 validation spike and cacheable-root design into the flat `docs/superpowers/archive/` directory. - Modify: every reference returned by `rg` for those filenames - [ ] **Step 1: Move the ESI dependency to workspace dependencies** @@ -347,7 +354,7 @@ Set `BID_DELAY=3` on both workflow harness invocations. In the script, branch ar Move both files into the flat `docs/superpowers/archive/` convention. Run: ```bash -rg -n "2026-08-10-1009-esi-validation-spike|2026-08-08-esi-cacheable-root-validation-design" docs crates +rg -n "1009-esi-validation-spike|esi-cacheable-root-validation-design" docs crates ``` Update every result, including relative links inside the archived files, until all links resolve to the archive paths. @@ -358,7 +365,7 @@ Run: `cd docs && npm run format` Run: `cd crates/trusted-server-js/lib && npm run format` -Run: `rg -n "docs/superpowers/(plans/2026-08-10-1009-esi-validation-spike|specs/2026-08-08-esi-cacheable-root-validation-design)" docs crates` +Run a stale-link search for either archived filename under the old `plans/` or `specs/` directory. Expected: formatters PASS and the stale-path search returns no matches. @@ -372,6 +379,7 @@ git commit -m "Align ESI spike documentation and tooling" ### Task 9: Resolve the fingerprint suggestion against runtime lifetime **Files:** + - Inspect: `crates/trusted-server-adapter-fastly/src/app.rs` - Inspect: `crates/trusted-server-core/src/publisher.rs` - No planned code modification: expand scope only after proving a durable cross-request owner exists @@ -397,6 +405,7 @@ Do not create an empty commit. Link the `AppState` lifecycle evidence and explai ### Task 10: Full verification and review handoff **Files:** + - Inspect: all changed files - No code changes unless a verification failure exposes a regression; any fix starts a new RED/GREEN cycle. diff --git a/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md index 5b593a15d..7cea844b4 100644 --- a/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md +++ b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md @@ -2,7 +2,7 @@ **Date:** 2026-08-11 **Status:** Decision record. Supersedes the delivery half of the -[ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md); the cache half +[ESI validation spike](../archive/2026-08-10-1009-esi-validation-spike.md); the cache half stands. **Issue:** IABTechLab/trusted-server#1009 diff --git a/docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md b/docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md index dcb814dd9..32184f1fd 100644 --- a/docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md +++ b/docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md @@ -76,7 +76,7 @@ The remediation will: - document the local harness coupling at `build_seam_script`; - list `PlatformTemplateCache` in the platform module roster and consolidate exports; - run both C2 workflow invocations with `BID_DELAY=3`, keep the `first_body_byte < complete / 3` assertion, and stop after one diagnostic failure when probe timings are non-numeric instead of performing a second comparison with fabricated zero values; -- move `docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md` and `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` directly into `docs/superpowers/archive/`, then update every reference found by `rg` including links inside the moved documents. +- move both self-labeled historical #1009 documents directly into `docs/superpowers/archive/`, then update every reference found by `rg` including links inside the moved documents. ## Error Handling diff --git a/scripts/c2-local-test.sh b/scripts/c2-local-test.sh index cd987551a..ad2415cee 100755 --- a/scripts/c2-local-test.sh +++ b/scripts/c2-local-test.sh @@ -579,28 +579,29 @@ COMPLETE=$(echo "$B_LINE" | sed -n 's/.*complete=\([0-9]*\)ms.*/\1/p') if ! [[ "$FIRST_BODY" =~ ^[0-9]+$ && "$COMPLETE" =~ ^[0-9]+$ ]]; then bad "socket probe did not return numeric body timings: '$B_LINE'" - FIRST_BODY=0 - COMPLETE=0 +else + if [ "$MODE" = "inline" ]; then + check "inline delivers the article before the auction resolves" \ + "$(awk -v f="$FIRST_BODY" -v c="$COMPLETE" 'BEGIN { print (f < c / 3) ? "yes" : "no" }')" \ + "yes" + else + # The property the unit tests cannot reach: in-process there is no bid provider, so + # there is no auction to wait on and reordering the stream is unobservable. Here the + # bid endpoint really sleeps, so the first body byte either beats it or does not. + check "cache hit streams: the article is delivered before the auction resolves" \ + "$(awk -v f="$FIRST_BODY" -v c="$COMPLETE" 'BEGIN { print (f < c / 3) ? "yes" : "no" }')" \ + "yes" + fi + printf ' first body byte %sms, complete %sms\n\n' "$FIRST_BODY" "$COMPLETE" fi -if [ "$MODE" = "inline" ]; then - check "inline delivers the article before the auction resolves" \ - "$(awk -v f="$FIRST_BODY" -v c="$COMPLETE" 'BEGIN { print (f < c / 3) ? "yes" : "no" }')" \ - "yes" -else - # The property the unit tests cannot reach: in-process there is no bid provider, so - # there is no auction to wait on and reordering the stream is unobservable. Here the - # bid endpoint really sleeps, so the first body byte either beats it or does not. +if [ "$MODE" != "inline" ]; then # Guards a regression where assembly rewrote a reader's accepted gzip origin request # to identity, making the origin send ~674KB where it would have sent ~100KB. The # cache still stores identity; that does not require changing what this reader accepts. check "the origin fetch stays compressed" \ "$(grep -c 'served PLAINTEXT' "$WORK/origin.log" || true)" "0" - check "cache hit streams: the article is delivered before the auction resolves" \ - "$(awk -v f="$FIRST_BODY" -v c="$COMPLETE" 'BEGIN { print (f < c / 3) ? "yes" : "no" }')" \ - "yes" fi -printf ' first body byte %sms, complete %sms\n\n' "$FIRST_BODY" "$COMPLETE" info "Result" printf ' %d passed, %d failed\n\n' "$PASS" "$FAIL" diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 83ecbe369..74afb7e35 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -203,8 +203,14 @@ price_granularity = "dense" # Request headers (other than Accept-Encoding, whose supported content coding TS # decodes to identity before storage) that the origin may name in Vary. Values # are keyed losslessly and the origin response is checked for drift before -# storage. Never include Cookie: C2 templates must be reader-neutral. -# template_cache_vary = ["rsc", "next-router-prefetch"] +# storage. Every emitted Vary name must be covered or storage is refused. Never +# include Cookie: C2 templates must be reader-neutral. +# template_cache_vary = [ +# "rsc", +# "next-router-state-tree", +# "next-router-prefetch", +# "next-router-segment-prefetch", +# ] # Safety ceiling for one reader-neutral C2 template, in seconds. The origin must # still authorize shared freshness; TS uses the smaller of that remaining edge From 379c33d2be61456be04b945d9c1c389d2310692a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 15:54:52 +0530 Subject: [PATCH 360/395] Fix ESI design whitespace --- .../specs/2026-08-12-1009-esi-merge-hardening-design.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md b/docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md index 653960744..a66f5d177 100644 --- a/docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md +++ b/docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md @@ -1,7 +1,7 @@ # #1009 ESI Merge and Hardening Design -**Date:** 2026-08-12 -**Status:** Approved for implementation +**Date:** 2026-08-12 +**Status:** Approved for implementation **Branch:** `1009-esi-cacheable-root-spec` ## Goal From 24a855b6789493ad14705411073de00a79373c8f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 16:29:32 +0530 Subject: [PATCH 361/395] Design template cache terminology rename --- ...08-19-template-cache-terminology-design.md | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-19-template-cache-terminology-design.md diff --git a/docs/superpowers/specs/2026-08-19-template-cache-terminology-design.md b/docs/superpowers/specs/2026-08-19-template-cache-terminology-design.md new file mode 100644 index 000000000..8f5f99df0 --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-template-cache-terminology-design.md @@ -0,0 +1,66 @@ +# Template-cache terminology + +## Goal + +Replace the experimental shared-template cache's `C2` shorthand with names that tell an +operator or contributor what the component does. The preferred term is **template +cache**, and the public diagnostic header becomes `X-TS-Template-Cache`. + +## Scope + +Rename the active implementation and its interfaces consistently: + +- Rust types, constants, functions, variables, test modules, assertions, comments, and + log messages use `TemplateCache` or `template_cache` rather than `C2` or `c2`. +- The response header changes from `X-TS-C2-Cache` to `X-TS-Template-Cache`. The old + header is not emitted as an alias because the feature is an experimental #1009 spike, + not a stable interface. +- The local harness becomes `scripts/template-cache-local-test.sh`; CI, documentation, + and references move with it. +- All non-archived operator documentation, examples, plans, and specifications say + "template cache" rather than relying on the numbered cache taxonomy. +- Both opaque platform-key namespace components change from `ts-c2` to + `ts-template-cache`: the canonical hash-domain bytes and the rendered key prefix. + Fixed-length and key-format assertions move with them. The new namespace naturally + makes old entries miss after deployment. The existing template schema version does + not change because the transformed template bytes and assembly seam do not change. +- Active test fixtures such as `reserved-c2-seam`, diagnostic identifiers, assertions, + and comments are renamed even when they are not externally visible. + +Only three categories may retain the old spelling: + +1. the exact `` marker in the template schema-version + history; +2. before/after compatibility references in this migration design; and +3. documents already under `docs/superpowers/archive/`, which remain unchanged as + historical records. + +Unrelated `c2` substrings in creative fixtures, cookie or EC identifiers, lockfiles, +checksums, styling, or third-party content are outside scope. + +## Behaviour and compatibility + +This is a terminology change, not a cache-policy or assembly change. Header values, +eligibility decisions, TTL calculation, privacy enforcement, template bytes, and +miss/hit assembly remain identical. + +The deliberate compatibility effects are: + +1. Consumers of `X-TS-C2-Cache` must move to `X-TS-Template-Cache`. +2. Monitoring that matches `c2_template_cache` logs must move to `template_cache` logs. +3. Existing objects in the old opaque key namespace are not read. They expire normally; + the first request in the new namespace safely creates a new template. +4. Callers of `scripts/c2-local-test.sh` must use the new script path. + +No dual header, dual read, or script shim is warranted for an unmerged experimental +branch because each would preserve the terminology this change is removing. + +## Verification + +- Focused Rust tests cover diagnostic header values, cache keys, cache states, and + miss/hit assembly. +- The renamed local harness passes in `esi` and `inline` modes. +- Fastly tests and target-matched Clippy pass. +- Formatting passes for Rust, JavaScript, and documentation. +- A final search finds no active cache-related `C2`/`c2` naming outside the exact + exceptions listed above. From e935e141860605a0a411387c65b07e9f28fb1ffc Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 16:35:22 +0530 Subject: [PATCH 362/395] Use synthetic fixtures for per-render slot tests The known-per-render slot tests embedded a real GAM network id, ad-unit path, and div-id token. Replace them with the fictional network id already used elsewhere in the module, a generic placement path, and a synthetic token. The `rh-gam-kso` prefix stays: it is the shipped matcher constant under test, not fixture data. The token keeps the shape the matcher requires (eight or more leading digits followed by letters), so both the dynamic and all-digit branches are still exercised. --- .../src/commands/audit/generate/gpt_slots.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index a0ce98825..03837a136 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -1034,8 +1034,8 @@ mod tests { fn single_known_per_render_registry_slot_is_refused() { let discovered = discover_gpt_slots( &[registry_slot( - "/22558409563/autoblog.com_In-Article_Desktop_ESP_jfOmMslaux", - "rh-gam-kso_26332072TPTy2yC1wkhc_ei_inarticle_1", + "/123456789/site_in-article_desktop_1", + "rh-gam-kso_12345678AbCdEfGh_ei_inarticle_1", &[(300, 250)], )], &[], @@ -1047,39 +1047,39 @@ mod tests { discovered.slots.is_empty(), "one observation of a known per-render family must not be written literally" ); - assert_eq!(discovered.gam_network_id.as_deref(), Some("22558409563")); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); assert_known_per_render_warning(&discovered); } #[test] fn single_known_per_render_request_slot_is_refused() { let discovered = from_requests(&[request( - "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=22558409563%2Cautoblog.com_In-Article_Desktop_ESP_jfOmMslaux&dids=rh-gam-kso_26332072TPTy2yC1wkhc_ei_inarticle_1&prev_iu_szs=300x250", + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite_in-article_desktop_1&dids=rh-gam-kso_12345678AbCdEfGh_ei_inarticle_1&prev_iu_szs=300x250", )]); assert!(discovered.had_slot_evidence); assert!(discovered.slots.is_empty()); - assert_eq!(discovered.gam_network_id.as_deref(), Some("22558409563")); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); assert_known_per_render_warning(&discovered); } #[test] fn known_per_render_match_does_not_claim_arbitrary_vendor_ids() { assert_eq!( - known_per_render_div_prefix("rh-gam-kso_26332072TPTy2yC1wkhc_ei_inarticle_1"), + known_per_render_div_prefix("rh-gam-kso_12345678AbCdEfGh_ei_inarticle_1"), Some("rh-gam-kso") ); assert_eq!( - known_per_render_div_prefix("rh-gam-kso_26332072TPTy2yC1wkhc_ei_overlay_1-container"), + known_per_render_div_prefix("rh-gam-kso_12345678AbCdEfGh_ei_overlay_1-container"), Some("rh-gam-kso") ); for stable in [ "rh-gam-kso_stable_ei_inarticle_1", - "rh-gam-kso_26332072_ei_inarticle_1", - "rh-gam-kso_26332072TPTy2yC1wkhc_ei_sidebar_1", - "rh-gam-kso_26332072TPTy2yC1wkhc_ei_overlay_stable", - "rh-gam-kso_26332072TPTy2yC1wkhc_ei_overlay_", - "rh-gam-kso_26332072TPTy2yC1wkhc_ei_overlay_1_extra", + "rh-gam-kso_12345678_ei_inarticle_1", + "rh-gam-kso_12345678AbCdEfGh_ei_sidebar_1", + "rh-gam-kso_12345678AbCdEfGh_ei_overlay_stable", + "rh-gam-kso_12345678AbCdEfGh_ei_overlay_", + "rh-gam-kso_12345678AbCdEfGh_ei_overlay_1_extra", "rh-gam-kso-header", ] { assert_eq!( From 09b7e4a7fbbc6a588d862b6c262b4eb0ed953a0d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 16:35:51 +0530 Subject: [PATCH 363/395] Accept a same-host HTTPS upgrade during slot generation `Url::origin()` includes the scheme, so an `http://publisher.example/` entry URL that canonically redirects to `https://publisher.example/` was refused as a cross-origin root redirect, forcing operators to find and enter the canonical URL before generation could run. Reuse the verify path's `origin_changed`, which already permits only the same-host default-port `http:80` to `https:443` upgrade and still refuses host changes, port changes, and HTTPS downgrades. The host is the cookie boundary, so the upgrade leaves the trust boundary intact. --- .../src/commands/audit/ad_templates.rs | 6 +- .../src/commands/audit/generate/mod.rs | 83 ++++++++++++++++++- 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs index b36bd373f..72c1733e1 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -165,7 +165,11 @@ fn build_report( } /// Whether navigation left the requested URL's origin (scheme, host, or port). -fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { +/// +/// A same-host default-port `http:80` to `https:443` redirect is *not* a change: +/// the host is the cookie boundary, and that upgrade is the ordinary canonical +/// redirect. Host changes, port changes, and HTTPS downgrades all are. +pub(super) fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { if requested.host_str() != final_url.host_str() { return true; } diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 4cedacc5b..af1c1697c 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -20,6 +20,7 @@ use trusted_server_core::creative_opportunities::{ }; use url::Url; +use crate::commands::audit::ad_templates::origin_changed; use crate::commands::audit::generate::collector::AuditCollector; use crate::commands::audit::generate::slot_toml::{ render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, toml_string, @@ -556,7 +557,7 @@ pub(crate) fn run_update_slots( &mut report_progress, &mut |_, root| { root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); - if root_url.origin() != target_url.origin() { + if origin_changed(&target_url, &root_url) { return cli_error(format!( "refusing cross-origin root redirect from {} to {}; the requested origin is the audit and cookie trust boundary", target_url, root_url @@ -2040,6 +2041,86 @@ mod tests { ); } + #[test] + fn update_slots_accepts_a_same_host_https_upgrade() { + // The ordinary canonical redirect: an operator types the bare http URL + // and the site upgrades it. The host is unchanged, so the cookie and + // audit trust boundary is unchanged, and generation must not stall on it. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.requested_url = "http://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/".to_string(); + let collector = FakeCollector::new(collected); + + run_update_slots( + &UpdateSlotsRequest { + url: "http://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect("a same-host HTTPS upgrade should not be treated as cross-origin"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse config"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "evidence from the upgraded root should be written" + ); + } + + #[test] + fn update_slots_rejects_an_https_downgrade_root_redirect() { + // The mirror image of the accepted upgrade: same host, but dropping TLS + // leaves the requested trust boundary and must still be refused. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.final_url = "http://publisher.example/".to_string(); + let collector = FakeCollector::new(collected); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("an HTTPS downgrade must leave the requested trust boundary"); + + assert!(format!("{error:?}").contains("cross-origin")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "downgraded evidence must not rewrite the config" + ); + } + #[test] fn update_slots_requires_evidence_from_every_selected_profile() { let temp = TempDir::new().expect("should create temp dir"); From 7d21559d6a8a2c0a985c800f627691e51d6e8899 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 16:43:57 +0530 Subject: [PATCH 364/395] Plan template cache terminology rename --- .../2026-08-19-template-cache-terminology.md | 324 ++++++++++++++++++ ...08-19-template-cache-terminology-design.md | 3 +- 2 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/plans/2026-08-19-template-cache-terminology.md diff --git a/docs/superpowers/plans/2026-08-19-template-cache-terminology.md b/docs/superpowers/plans/2026-08-19-template-cache-terminology.md new file mode 100644 index 000000000..8f5bd169d --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-template-cache-terminology.md @@ -0,0 +1,324 @@ +# Template-cache terminology Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace active shared-template-cache `C2`/`c2` terminology with `TemplateCache`/`template_cache`, migrate the public diagnostic header and opaque key namespace, and rename the local harness without changing cache behavior. + +**Architecture:** Keep the existing publisher, platform-cache, Fastly adapter, and ESI assembly boundaries. Change the key hash-domain bytes and rendered prefix together, rename the response-state API and diagnostics together, and update all active callers and documentation; do not change schema version, template bytes, eligibility, freshness, privacy, or assembly behavior. The migration design itself remains the source of the permitted old-spelling compatibility references, and the archived documents remain historical records. + +**Tech Stack:** Rust 2024 (`trusted-server-core`, Fastly adapter, wasm32-wasip1 tests), Bash, GitHub Actions YAML, TOML, Markdown, and the repository’s Cargo/npm verification aliases. + +--- + +## File map and terminology boundary + +Implementation and tests: + +- Modify `crates/trusted-server-core/src/platform/template_cache.rs`: replace the `ts-c2` canonical hash-domain and rendered key prefix with `ts-template-cache`; rewrite taxonomy comments; update/add deterministic namespace tests. Keep the exact schema-history marker `` unchanged. +- Modify `crates/trusted-server-core/src/publisher.rs`: rename `HEADER_X_TS_C2_CACHE`, `C2ResponseState`, `C2BypassReason`, `C2CachePolicy`, `set_c2_response_state`, `request_bypasses_c2`, `c2_bypass_reason`, `c2_cache_ttl`, local variables, test modules, assertions, comments, and `c2_template_cache` log messages; emit only `x-ts-template-cache` and test that the old header is absent. +- Modify `crates/trusted-server-core/src/html_processor.rs`: rename the active `reserved-c2-seam` test fixture to a template-cache seam name and update its assertions/comments. This fixture is not the schema-history marker exception. +- Modify `crates/trusted-server-core/src/creative_opportunities.rs`, `crates/trusted-server-core/src/platform/types.rs`, and `crates/trusted-server-core/src/response_privacy.rs`: rewrite active cache comments and validation/test messages to “template cache” terminology. +- Modify `crates/trusted-server-adapter-fastly/src/template_cache.rs` and `crates/trusted-server-adapter-fastly/src/esi_assembly.rs`: rewrite active comments and the legacy-read log to `template_cache`; do not alter Fastly cache operations or ESI behavior. + +Harness, CI, and operator material: + +- Rename `scripts/c2-local-test.sh` to `scripts/template-cache-local-test.sh`; update its function name, header parser, log regexes, comments, and usage text. +- Modify `.github/workflows/test.yml` so the step name and both invocations call `scripts/template-cache-local-test.sh`. +- Modify `trusted-server.example.toml` and `docs/guide/configuration.md` so operator-facing prose, diagnostics, and harness commands say “template cache” and use `X-TS-Template-Cache`. +- Modify these non-archived active plans/specifications where the occurrences describe this cache: `docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md`, `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md`, `docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md`, `docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md`, `docs/superpowers/plans/2026-08-19-pr-1013-review-remediation.md`, `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md`, `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md`, `docs/superpowers/specs/2026-08-14-1009-esi-parser-assembly-design.md`, and `docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md`. Preserve unrelated `c2` substrings such as checksums, cookie/EC identifiers, creative IDs, and third-party fixture content. +- Do not edit `docs/superpowers/archive/**`. Do not rewrite the exact schema-history marker. Leave `docs/superpowers/specs/2026-08-19-template-cache-terminology-design.md` as the migration record of the old/new names and compatibility effects; its before/after references are an explicit exception. + +The implementation worker should use `@superpowers:test-driven-development` for the focused contract changes, `@superpowers:subagent-driven-development` or `@superpowers:executing-plans` for this task sequence, and `@superpowers:verification-before-completion` before claiming completion. + +### Task 1: Establish failing key-namespace and public-header contracts + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` (test module near `rendered_key_is_fixed_size_and_contains_no_request_material`) +- Modify: `crates/trusted-server-core/src/publisher.rs` (the existing cold/warm end-to-end response-state test) + +- [ ] **Step 1: Run the current focused baseline.** + +Run: + +```bash +cargo test-fastly template_cache::tests::rendered_key_is_fixed_size_and_contains_no_request_material +cargo test-fastly publisher::c2_end_to_end_tests::a_second_request_is_served_from_the_cache_without_touching_the_origin +``` + +Expected: both commands PASS against the old `ts-c2-v4-...` key and `x-ts-c2-cache` header, establishing that the existing behavior is green before changing expected contracts. + +- [ ] **Step 2: Add the deterministic new namespace expectation first.** + +In `platform/template_cache.rs`, change the fixed-length expectation to use `ts-template-cache-v{TEMPLATE_SCHEMA_VERSION}-`, assert that the rendered key starts with that prefix, and add an exact fixture assertion for `key()`: + +```rust +assert_eq!( + key().to_cache_key(), + "ts-template-cache-v4-54431eb4ea82644d6378717a8c3f18302fafbf739e684598da79e392b16900a6" +); +``` + +This exact value proves both the visible prefix and the canonical hash-domain bytes changed; it must not be replaced by a length-only assertion. + +- [ ] **Step 3: Add the public-header expectation first.** + +In the cold/warm publisher test, read `x-ts-template-cache` using `HeaderName::from_static` and assert the old `x-ts-c2-cache` header is absent on both the cold and warm responses. Keep the existing `miss-stored` and `hit` values and origin/body assertions unchanged. Use the new raw header name in this test before renaming the production constant so the test compiles and fails at the observable contract. + +- [ ] **Step 4: Run the focused contracts and verify they fail for the intended old values.** + +Run: + +```bash +cargo test-fastly template_cache::tests::rendered_key_is_fixed_size_and_contains_no_request_material +cargo test-fastly publisher::c2_end_to_end_tests::a_second_request_is_served_from_the_cache_without_touching_the_origin +``` + +Expected: the key test FAILS with the old `ts-c2-v4-...` output (including the old digest), and the publisher test FAILS because the response still emits `x-ts-c2-cache` instead of `x-ts-template-cache`. No policy, body, or cache-state assertion should fail for another reason. + +- [ ] **Step 5: Commit the red contract tests.** + +Do not commit source implementation changes yet. Commit only the two focused test expectation changes: + +```bash +git add crates/trusted-server-core/src/platform/template_cache.rs crates/trusted-server-core/src/publisher.rs +git commit -m "Specify template cache namespace and header" +``` + +### Task 2: Migrate the opaque template-cache key namespace + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Test: `crates/trusted-server-core/src/platform/template_cache.rs` + +- [ ] **Step 1: Replace both namespace components and terminology in the implementation.** + +Change only the namespace inputs/labels and active prose: hash `b"ts-template-cache"` instead of `b"ts-c2"`, render `ts-template-cache-v{schema_version}-{digest}`, and describe the shared transformed template without the C1/C2/C3 numbered taxonomy. Keep `TEMPLATE_SCHEMA_VERSION` at `4`, keep the exact historical v3 marker, keep surrogate keys, and keep all key fields/order and hash algorithm unchanged. + +- [ ] **Step 2: Run the key-focused suite.** + +Run: + +```bash +cargo test-fastly template_cache +``` + +Expected: PASS, including the exact deterministic namespace assertion, fixed-length/key-format assertion, all-field-distinctness tests, delimiter-collision test, Vary case/order tests, metadata tests, and reservation tests. The old namespace must not be accepted as an alias or read path. + +- [ ] **Step 3: Commit the namespace boundary.** + +```bash +git add crates/trusted-server-core/src/platform/template_cache.rs +git commit -m "Move template cache keys to named namespace" +``` + +### Task 3: Rename publisher state, policy APIs, logs, and public diagnostics + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Rename the response-state API and all active internal identifiers.** + +Use `TemplateCacheResponseState`, `HEADER_X_TS_TEMPLATE_CACHE`, `set_template_cache_response_state`, `TemplateCacheBypassReason`, `TemplateCachePolicy`, `request_bypasses_template_cache`, `template_cache_bypass_reason`, and `template_cache_ttl`. Rename local `c2_*` variables and the `c2_store_authorization_tests`, `c2_end_to_end_tests`, and `c2_gate_tests` modules to `template_cache_*`. Replace every active `c2_template_cache` log prefix with `template_cache`; preserve the same bounded values (`hit`, `miss-stored`, `miss-store-error`, `miss-reserved`, `bypass-request`, `bypass-response`, `unsupported`, `invalid`, `backend-error`). Rewrite comments/assertion messages to “template cache” without changing logic. + +- [ ] **Step 2: Emit only the new public header.** + +Make the renamed setter insert `HeaderValue::from_static(state.as_str())` under `x-ts-template-cache`. Do not emit `x-ts-c2-cache` as an alias. Update all publisher tests that use the constant or literal to the renamed constant/new literal, while retaining the explicit old-header-absent assertion from Task 1. + +- [ ] **Step 3: Run the focused publisher suites.** + +Run: + +```bash +cargo test-fastly template_cache_store_authorization_tests +cargo test-fastly template_cache_end_to_end_tests +cargo test-fastly template_cache_gate_tests +``` + +Expected: PASS. Cold, warm, reserved, bypass, unsupported, invalid, and backend-error states retain their existing values; miss/hit assembly, origin counts, privacy headers, diagnostics bypass, policy gates, and body identity remain unchanged. The new header is present for each relevant state and the old header is absent. + +- [ ] **Step 4: Commit the publisher boundary.** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Name template cache diagnostics" +``` + +### Task 4: Finish active Rust terminology and seam fixtures + +**Files:** + +- Modify: `crates/trusted-server-core/src/html_processor.rs` +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Modify: `crates/trusted-server-core/src/response_privacy.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` + +- [ ] **Step 1: Rename the active seam fixture and supporting prose.** + +Change `reserved-c2-seam` to `reserved-template-cache-seam` in the HTML processor test and its source/collision assertions. Rewrite comments and rustdoc that call the shared transformed template “C2”; use “template cache” or “shared transformed template.” Do not touch the exact `ts-c2-v3` schema-history marker in `platform/template_cache.rs`. + +- [ ] **Step 2: Rename Fastly adapter log/comment terminology.** + +Change the legacy read warning to `template_cache legacy read failed` and update Fastly/ESI rustdoc. Do not change error classification, cache key construction (which already consumes the core key), transaction behavior, or assembly output. + +- [ ] **Step 3: Run the focused supporting suites.** + +Run: + +```bash +cargo test-fastly html_processor +cargo test-fastly template_cache +cargo test-fastly publisher +``` + +Expected: PASS; the renamed fixture still proves the transform-owned terminal seam, the key suite still proves the new namespace, and publisher behavior remains unchanged apart from terminology/header names. + +- [ ] **Step 4: Commit the remaining Rust terminology.** + +```bash +git add crates/trusted-server-core/src/html_processor.rs crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/platform/types.rs crates/trusted-server-core/src/response_privacy.rs crates/trusted-server-adapter-fastly/src/template_cache.rs crates/trusted-server-adapter-fastly/src/esi_assembly.rs +git commit -m "Describe shared templates consistently" +``` + +### Task 5: Rename the local harness and its CI caller + +**Files:** + +- Rename: `scripts/c2-local-test.sh` → `scripts/template-cache-local-test.sh` +- Modify: `scripts/template-cache-local-test.sh` +- Modify: `.github/workflows/test.yml` + +- [ ] **Step 1: Rename the script without creating a compatibility shim.** + +Use `git mv scripts/c2-local-test.sh scripts/template-cache-local-test.sh`. Update usage comments, `c2_state` to `template_cache_state`, the `x-ts-c2-cache` parser to `x-ts-template-cache`, all `c2_template_cache` log patterns to `template_cache`, and prose describing the inert marker. Keep `esi` and `inline` argument behavior and all timing/body/origin-count assertions intact. + +- [ ] **Step 2: Update CI callers.** + +Rename the workflow step to “Run template cache ESI local harness” and update both CI commands to `BID_DELAY=3 ./scripts/template-cache-local-test.sh esi` and `BID_DELAY=3 ./scripts/template-cache-local-test.sh inline`. + +- [ ] **Step 3: Run shell and harness verification.** + +Run: + +```bash +bash -n scripts/template-cache-local-test.sh +BID_DELAY=3 ./scripts/template-cache-local-test.sh esi +BID_DELAY=3 ./scripts/template-cache-local-test.sh inline +``` + +Expected: syntax check PASS; both harness modes PASS, with cold `miss-stored`, warm `hit`, new `X-TS-Template-Cache` parsing, expected origin counts, seam/assembly integrity, and no old-header/log matches. If local Viceroy prerequisites are unavailable, record that environmental block explicitly and run the same commands in CI before completion; do not add a legacy script shim. + +- [ ] **Step 4: Commit the harness/CI boundary.** + +```bash +git status --short scripts .github/workflows/test.yml +git add -A -- scripts .github/workflows/test.yml +git commit -m "Rename template cache local harness" +``` + +### Task 6: Update active operator documentation, examples, plans, and specs + +**Files:** + +- Modify: `trusted-server.example.toml` +- Modify: `docs/guide/configuration.md` +- Modify: the nine active plans/specs listed in the file map + +- [ ] **Step 1: Update the operator example and guide.** + +Replace numbered-cache prose with “template cache,” update all diagnostic examples to `X-TS-Template-Cache`, and update harness commands to `scripts/template-cache-local-test.sh`. Preserve the configuration keys, safety caveats, bounded state values, rollback instructions, `ts-template` surrogate key, and all behavior descriptions. Explain raw origin caching/final assembly by those names where an old C1/C2 taxonomy was used. + +- [ ] **Step 2: Update active plans/specs mechanically but semantically.** + +Rename cache-related `C2`, `c2_*`, `x-ts-c2-cache`, and `scripts/c2-local-test.sh` references to the named terminology, including completed checklist text and historical findings. Rewrite sentences that distinguish cache layers in terms of raw origin bytes, shared template cache, and final assembled response. Leave unrelated IDs, hashes, cookie/EC identifiers, and third-party content unchanged. + +- [ ] **Step 3: Verify documentation formatting and active references.** + +Run: + +```bash +cd docs && npm run format +cd .. +rg -n -i --glob '!docs/superpowers/archive/**' --glob '!docs/superpowers/specs/2026-08-19-template-cache-terminology-design.md' --glob '!docs/superpowers/plans/2026-08-19-template-cache-terminology.md' 'X-TS-C2-Cache|x-ts-c2-cache|ts-c2|c2_template_cache|C2Response|C2Bypass|C2Cache|c2_bypass|c2_cache|c2-local-test|reserved-c2-seam|\bC2\b|\bc2\b' crates/trusted-server-core/src/platform/template_cache.rs crates/trusted-server-core/src/publisher.rs crates/trusted-server-core/src/html_processor.rs crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/platform/types.rs crates/trusted-server-core/src/response_privacy.rs crates/trusted-server-adapter-fastly/src scripts trusted-server.example.toml docs/guide docs/superpowers/plans docs/superpowers/specs +``` + +Expected: the only result is the exact retained +`` schema-history marker in +`platform/template_cache.rs`. Inspect that single result rather than weakening the +search. The excluded migration design and implementation plan may retain their explicit +old/new compatibility references; archived documents and unrelated substrings are not +migration failures. + +- [ ] **Step 4: Commit the documentation boundary.** + +```bash +git add trusted-server.example.toml docs/guide/configuration.md docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md docs/superpowers/plans/2026-08-08-1009-measurement-findings.md docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md docs/superpowers/plans/2026-08-19-pr-1013-review-remediation.md docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md docs/superpowers/specs/2026-08-14-1009-esi-parser-assembly-design.md docs/superpowers/specs/2026-08-19-pr-1013-review-remediation-design.md +git commit -m "Use template cache terminology in documentation" +``` + +### Task 7: Run full verification and review the migration diff + +**Files:** + +- Test/verify: all files changed by Tasks 1–6 + +- [ ] **Step 1: Run Rust formatting and target-matched tests.** + +Run: + +```bash +cargo fmt --all -- --check +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +cargo fmt --manifest-path crates/trusted-server-integration-tests/Cargo.toml -- --check +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +``` + +Expected: all commands PASS. The Fastly suite is the authoritative shared-template-cache test target; Axum and Cloudflare confirm the terminology/API changes do not break adapters that use the unavailable-cache fallback. + +- [ ] **Step 2: Run target-matched Clippy and JS tests/formatting.** + +Run: + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy --manifest-path crates/trusted-server-integration-tests/Cargo.toml --all-targets -- -D warnings +cd crates/trusted-server-js/lib && npx vitest run && npm run format && node build-all.mjs +cd ../../.. +``` + +Expected: all commands PASS with no new warnings, and JavaScript tests/formatting remain green; no JS behavior should have changed. + +- [ ] **Step 3: Re-run the renamed harness and documentation search.** + +Run: + +```bash +BID_DELAY=3 ./scripts/template-cache-local-test.sh esi +BID_DELAY=3 ./scripts/template-cache-local-test.sh inline +rg -n -i --glob '!docs/superpowers/archive/**' --glob '!docs/superpowers/specs/2026-08-19-template-cache-terminology-design.md' --glob '!docs/superpowers/plans/2026-08-19-template-cache-terminology.md' 'X-TS-C2-Cache|x-ts-c2-cache|ts-c2|c2_template_cache|C2Response|C2Bypass|C2Cache|c2_bypass|c2_cache|c2-local-test|reserved-c2-seam|\bC2\b|\bc2\b' crates/trusted-server-core/src/platform/template_cache.rs crates/trusted-server-core/src/publisher.rs crates/trusted-server-core/src/html_processor.rs crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/platform/types.rs crates/trusted-server-core/src/response_privacy.rs crates/trusted-server-adapter-fastly/src scripts trusted-server.example.toml docs/guide docs/superpowers/plans docs/superpowers/specs +git diff --check +git status --short +``` + +Expected: both harness modes PASS; the active-cache search returns only the exact +retained v3 schema-history marker; `git diff --check` PASS; and `git status --short` is +empty (no stale `scripts/c2-local-test.sh`, generated artifacts, or unrelated edits). +Confirm the only other retained old spellings live in the excluded migration design, +implementation plan, archived records, and explicitly unrelated substrings. + +- [ ] **Step 4: Review the final diff before handoff.** + +Use `git diff HEAD~6..HEAD --stat` and `git diff HEAD~6..HEAD --` (adjust the commit range if additional logical commits were made) to confirm the changes are terminology-only: no schema-version bump, no dual header, no old-namespace read, no policy/TTL/eligibility change, no template-byte change, and no script shim. Follow `@superpowers:verification-before-completion` and report command evidence before claiming completion. diff --git a/docs/superpowers/specs/2026-08-19-template-cache-terminology-design.md b/docs/superpowers/specs/2026-08-19-template-cache-terminology-design.md index 8f5f99df0..80298cdc5 100644 --- a/docs/superpowers/specs/2026-08-19-template-cache-terminology-design.md +++ b/docs/superpowers/specs/2026-08-19-template-cache-terminology-design.md @@ -31,7 +31,8 @@ Only three categories may retain the old spelling: 1. the exact `` marker in the template schema-version history; -2. before/after compatibility references in this migration design; and +2. before/after compatibility references in this migration design and its implementation + plan; and 3. documents already under `docs/superpowers/archive/`, which remain unchanged as historical records. From c2cd68c23c53b6a8bd9b9a3e113a50fa4e0b81a3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 16:51:38 +0530 Subject: [PATCH 365/395] Name template cache namespace and diagnostics --- .../src/platform/template_cache.rs | 34 +- crates/trusted-server-core/src/publisher.rs | 579 ++++++++++-------- 2 files changed, 330 insertions(+), 283 deletions(-) diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index 35823c1c4..76137201c 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -1,4 +1,4 @@ -//! The shared transformed-template cache (C2) for the #1009 ESI validation spike. +//! The shared transformed-template cache for the #1009 ESI validation spike. //! //! Three caches are in play and conflating them is what produced the original wrong //! conclusion in the design doc, so this module names which one it is: @@ -6,10 +6,10 @@ //! | Cache | Contents | Owner | //! | ----- | --------------------------------- | ------------------------------ | //! | C1 | raw origin bytes | Fastly read-through. Not this. | -//! | C2 | post-`lol_html`, pre-assembly | **This module.** | -//! | C3 | final per-user assembled response | **Must never exist.** | +//! | Template cache | post-`lol_html`, pre-assembly | **This module.** | +//! | Final response | final per-user assembled response | **Must never exist.** | //! -//! C2 holds a *shared template*: no per-user bytes, and no decisions that depend on +//! The template cache holds a *shared template*: no per-user bytes, and no decisions that depend on //! the request. What may and may not live in it is //! [§6.7 of the design doc](../../../../docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md), //! and the invariant is enforced by the rendered-document byte-identity tests in @@ -36,7 +36,7 @@ use crate::creative_opportunities::AssemblyMode; /// | 4 | Marker is the shorter, accurate [`AD_ASSEMBLY_SEAM`](crate::publisher::AD_ASSEMBLY_SEAM) | pub const TEMPLATE_SCHEMA_VERSION: u32 = 4; -/// Surrogate key attached to every template so an incident can purge C2 globally. +/// Surrogate key attached to every template so an incident can purge the template cache globally. pub const TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY: &str = "ts-template"; /// Inputs that select one cached template. @@ -89,7 +89,7 @@ impl TemplateCacheKey { } let mut canonical = Vec::new(); - push(&mut canonical, b"ts-c2"); + push(&mut canonical, b"ts-template-cache"); push(&mut canonical, &self.schema_version.to_be_bytes()); push( &mut canonical, @@ -122,7 +122,11 @@ impl TemplateCacheKey { } let digest = sha2::Sha256::digest(canonical); - format!("ts-c2-v{}-{}", self.schema_version, hex::encode(digest)) + format!( + "ts-template-cache-v{}-{}", + self.schema_version, + hex::encode(digest) + ) } /// Surrogate keys to attach at insert, for purge-based rollback. @@ -196,7 +200,7 @@ pub const REPLAYABLE_POLICY_HEADERS: &[&str] = &[ /// reader input. This assumes those origin variants differ only by HTTP content coding; /// operators must leave ESI disabled if an origin changes document semantics instead. /// Without this carve-out, the ordinary declaration sent by any compressing origin reads -/// as an uncovered gap and disqualifies the response, so **C2 would never cache anything +/// as an uncovered gap and disqualifies the response, so **the template cache would never store anything /// against a real origin** unless the operator redundantly listed a header the transform /// already normalizes. Found by review before it could make the spike measure a hit rate /// of approximately zero and read that as a result. @@ -331,7 +335,7 @@ impl VarySpec { /// safe. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TemplateMetadata { - /// Encoding of the stored bytes. C2 writes only `identity`; retaining the field in + /// Encoding of the stored bytes. The template cache writes only `identity`; retaining the field in /// metadata makes corrupt or stale representations fail validation on read. pub content_encoding: String, /// Content type to rebuild the response with. @@ -587,7 +591,7 @@ impl Drop for TemplateCacheReservation { if let Some(inner) = self.inner.take() && let Err(err) = inner.cancel() { - log::warn!("c2_template_cache reservation cancellation failed: {err}"); + log::warn!("template_cache reservation cancellation failed: {err}"); } } } @@ -635,7 +639,7 @@ pub trait PlatformTemplateCache: Send + Sync { /// /// This compatibility default exists for implementations with no transactional /// reservation support. It reports ordinary cold misses as `Unsupported`; an - /// adapter that supports C2 reservations must override it so cold requests can + /// adapter that supports template-cache reservations must override it so cold requests can /// return [`TemplateCacheLookup::Reserved`]. async fn lookup_or_reserve( &self, @@ -656,7 +660,7 @@ pub trait PlatformTemplateCache: Send + Sync { /// Store a template. /// - /// Callers must not call this without having consulted the C2 eligibility gate + /// Callers must not call this without having consulted the template-cache eligibility gate /// first: this method stores what it is given and cannot tell a shared template /// from a per-user one. async fn put( @@ -876,9 +880,11 @@ mod tests { fn rendered_key_is_fixed_size_and_contains_no_request_material() { let rendered = key().to_cache_key(); assert_eq!( - rendered.len(), - format!("ts-c2-v{TEMPLATE_SCHEMA_VERSION}-").len() + 64 + rendered, + "ts-template-cache-v4-54431eb4ea82644d6378717a8c3f18302fafbf739e684598da79e392b16900a6" ); + assert!(rendered.starts_with("ts-template-cache-v4-")); + assert_eq!(rendered.len(), 85); for sensitive in ["example.com", "/news/article", "rsc", "abc123"] { assert!( !rendered.contains(sensitive), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 0cba0bba3..174783f18 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -75,11 +75,11 @@ use crate::streaming_replacer::create_url_replacer; const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); -const HEADER_X_TS_C2_CACHE: &str = "x-ts-c2-cache"; +const HEADER_X_TS_TEMPLATE_CACHE: &str = "x-ts-template-cache"; const HEADER_X_TS_ASSEMBLY: &str = "x-ts-assembly"; #[derive(Clone, Copy, PartialEq, Eq)] -enum C2ResponseState { +enum TemplateCacheResponseState { Hit, MissReserved, MissStored, @@ -91,7 +91,7 @@ enum C2ResponseState { BackendError, } -impl C2ResponseState { +impl TemplateCacheResponseState { const fn as_str(self) -> &'static str { match self { Self::Hit => "hit", @@ -107,9 +107,12 @@ impl C2ResponseState { } } -fn set_c2_response_state(response: &mut Response, state: C2ResponseState) { +fn set_template_cache_response_state( + response: &mut Response, + state: TemplateCacheResponseState, +) { response.headers_mut().insert( - HEADER_X_TS_C2_CACHE, + HEADER_X_TS_TEMPLATE_CACHE, HeaderValue::from_static(state.as_str()), ); } @@ -739,7 +742,7 @@ async fn process_response_streaming_async( ); let input_compression = Compression::from_content_encoding(¶ms.content_encoding); - // A C2 template is always identity bytes. Decode during the transform instead of + // A template-cache response is always identity bytes. Decode during the transform instead of // recompressing and immediately decoding the entire buffered result afterwards. let output_compression = if params.template_cache_key.is_some() { Compression::None @@ -1203,7 +1206,7 @@ struct HtmlStreamProcessorParams<'a> { /// /// It does not leak today even without this gate, but only by coincidence: /// `requires_private_no_store()` is a strict superset of the conditions under which -/// a script is emitted, and that stamp lands before the C2 gate reads response +/// a script is emitted, and that stamp lands before the template cache gate reads response /// headers, so the gate refuses. Two independent conditions that happen to align, /// with nothing enforcing the relationship. This makes the guarantee explicit; /// `requires_private_no_store_is_a_superset_of_injection` keeps the coincidence as a @@ -1375,7 +1378,7 @@ pub enum PublisherResponse { /// Parameters for [`process_response_streaming`]. params: Box, }, - /// A shared template read from C2, to be assembled on the way out. + /// A shared template read from template cache, to be assembled on the way out. /// /// Distinct from [`Self::Stream`] because the bytes are **already transformed** — /// running them through `lol_html` again would inject a second tsjs ` "#; +/// Rendering owner for selected APS bids. +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ApsRenderingMode { + /// Render through Trusted Server's opaque static renderer route. + #[default] + TrustedServer, + /// Delegate rendering to the publisher's explicit browser hook. + PublisherNative, +} + /// Configuration for the APS `OpenRTB` integration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] #[validate(schema(function = "validate_inventory_identity_override"))] @@ -141,6 +152,9 @@ pub struct ApsConfig { /// Whether APS script creatives are eligible before winner selection. #[serde(default)] pub allow_script_creatives: bool, + /// Rendering owner for selected APS bids. + #[serde(default)] + pub rendering_mode: ApsRenderingMode, /// APS-authorized inventory domain used instead of the deployment hostname. #[serde(default, skip_serializing_if = "Option::is_none")] #[validate(custom(function = "validate_inventory_domain"))] @@ -314,6 +328,7 @@ impl Default for ApsConfig { timeout_ms: default_timeout_ms(), debug: false, allow_script_creatives: false, + rendering_mode: ApsRenderingMode::TrustedServer, inventory_domain: None, inventory_page_origin: None, } @@ -1184,7 +1199,9 @@ impl AuctionProvider for ApsAuctionProvider { } #[derive(Debug)] -struct ApsRendererIntegration; +struct ApsRendererIntegration { + rendering_mode: ApsRenderingMode, +} #[async_trait(?Send)] impl IntegrationProxy for ApsRendererIntegration { @@ -1193,7 +1210,10 @@ impl IntegrationProxy for ApsRendererIntegration { } fn routes(&self) -> Vec { - vec![IntegrationEndpoint::get(APS_RENDERER_ROUTE)] + (self.rendering_mode == ApsRenderingMode::TrustedServer) + .then(|| IntegrationEndpoint::get(APS_RENDERER_ROUTE)) + .into_iter() + .collect() } async fn handle( @@ -1225,6 +1245,22 @@ impl IntegrationProxy for ApsRendererIntegration { } } +impl IntegrationHeadInjector for ApsRendererIntegration { + fn integration_id(&self) -> &'static str { + APS_INTEGRATION_ID + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + (self.rendering_mode == ApsRenderingMode::PublisherNative) + .then(|| { + "" + .to_string() + }) + .into_iter() + .collect() + } +} + /// Register the APS static renderer endpoint when APS is enabled. /// /// # Errors @@ -1233,16 +1269,21 @@ impl IntegrationProxy for ApsRendererIntegration { pub fn register( settings: &Settings, ) -> Result, Report> { - let Some(_config) = settings.integration_config::(APS_INTEGRATION_ID)? else { + let Some(config) = settings.integration_config::(APS_INTEGRATION_ID)? else { return Ok(None); }; - let integration = Arc::new(ApsRendererIntegration); - Ok(Some( - IntegrationRegistration::builder(APS_INTEGRATION_ID) - .with_proxy(integration) - .without_js() - .build(), - )) + let integration = Arc::new(ApsRendererIntegration { + rendering_mode: config.rendering_mode, + }); + let registration = IntegrationRegistration::builder(APS_INTEGRATION_ID) + .without_js() + .with_head_injector(integration.clone()); + let registration = if config.rendering_mode == ApsRenderingMode::TrustedServer { + registration.with_proxy(integration) + } else { + registration + }; + Ok(Some(registration.build())) } /// Register the APS auction provider when enabled. @@ -1273,6 +1314,7 @@ mod tests { UserInfo, }; use crate::consent::ConsentContext; + use crate::integrations::IntegrationDocumentState; use crate::openrtb::{Eid, Uid}; use crate::platform::GeoInfo; use crate::platform::test_support::{ @@ -1289,6 +1331,7 @@ mod tests { timeout_ms: 800, debug: false, allow_script_creatives: false, + rendering_mode: ApsRenderingMode::TrustedServer, inventory_domain: None, inventory_page_origin: None, } @@ -1405,6 +1448,7 @@ mod tests { assert!(!canonical.debug); assert!(debug.debug); assert!(!canonical.allow_script_creatives); + assert_eq!(canonical.rendering_mode, ApsRenderingMode::TrustedServer); assert!(canonical.endpoint.ends_with("/e/pb/bid")); } @@ -1466,6 +1510,14 @@ mod tests { })) .is_err() ); + assert!( + serde_json::from_value::(json!({ + "account_id": "example-account", + "rendering_mode": "unsupported" + })) + .is_err(), + "should reject an unknown APS rendering mode" + ); for endpoint in [ "http://aps.example/e/pb/bid", "https://", @@ -2319,7 +2371,9 @@ mod tests { #[test] fn registers_and_serves_only_static_renderer_route() { - let integration = ApsRendererIntegration; + let integration = ApsRendererIntegration { + rendering_mode: ApsRenderingMode::TrustedServer, + }; let routes = integration.routes(); assert_eq!(routes.len(), 1, "should register one route"); assert_eq!(routes[0].method, Method::GET); @@ -2374,9 +2428,55 @@ mod tests { assert_eq!(registration.integration_id, APS_INTEGRATION_ID); assert_eq!(registration.proxies.len(), 1); + assert_eq!(registration.head_injectors.len(), 1); assert!(registration.js_disabled); } + #[test] + fn publisher_native_config_registers_hook_mode_without_renderer_route() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + APS_INTEGRATION_ID, + &json!({ + "enabled": true, + "account_id": "example-account", + "rendering_mode": "publisher_native" + }), + ) + .expect("should insert native APS config"); + + let registration = register(&settings) + .expect("should register APS") + .expect("should return enabled registration"); + assert!( + registration.proxies.is_empty(), + "should not register the static renderer" + ); + assert_eq!(registration.head_injectors.len(), 1); + + let integration = ApsRendererIntegration { + rendering_mode: ApsRenderingMode::PublisherNative, + }; + assert!( + integration.routes().is_empty(), + "should expose no renderer route" + ); + let document_state = IntegrationDocumentState::default(); + let context = IntegrationHtmlContext { + request_host: "publisher.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + assert_eq!( + integration.head_inserts(&context), + vec![""], + "should inject only the native-mode marker" + ); + } + #[test] fn config_without_enabled_does_not_register_provider_or_renderer() { let mut settings = create_test_settings(); diff --git a/crates/trusted-server-js/lib/src/core/request.ts b/crates/trusted-server-js/lib/src/core/request.ts index df3fee6a6..c05070a9d 100644 --- a/crates/trusted-server-js/lib/src/core/request.ts +++ b/crates/trusted-server-js/lib/src/core/request.ts @@ -1,5 +1,5 @@ // Request orchestration for tsjs: unified auction endpoint with iframe-based creative rendering. -import { renderApsCreative } from '../integrations/aps/render'; +import { dispatchApsRendering, renderApsCreative } from '../integrations/aps/render'; import { buildAdRequest, sendAuction } from './auction'; import { collectContext } from './context'; @@ -52,7 +52,13 @@ export function requestAds( for (const bid of bids) { if (!bid.impid) continue; if (bid.renderer) { - renderApsCreative({ slotId: bid.impid, renderer: bid.renderer }); + void Promise.resolve( + dispatchApsRendering({ + slotId: bid.impid, + renderer: bid.renderer, + trustedServer: (renderer) => renderApsCreative({ slotId: bid.impid, renderer }), + }) + ); continue; } if (!bid.adm) { diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 0c68d43fe..192b82033 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -68,6 +68,21 @@ export interface ApsRendererV1 { export type AuctionBidRenderer = ApsRendererV1; +/** Explicit acknowledgement returned by the opt-in publisher-native APS hook. */ +export interface ApsNativeRendererResult { + accepted: boolean; + reason?: string; +} + +/** Publisher-owned rendering seam for a fully validated APS descriptor. */ +export interface ApsNativeRendererHook { + render(input: { + version: 1; + slotId: string; + renderer: ApsRendererV1; + }): ApsNativeRendererResult | Promise; +} + /** A client-side Prebid bid's generated ad ID bound to its APS render capability. */ export interface ApsPrebidRendererEntry { adUnitCode: string; @@ -388,6 +403,8 @@ export interface TsjsApi { * `hb_adid`. The Universal Creative bridge consumes each entry at most once. */ apsPrebidRenderers?: Record; + /** Opt-in publisher-owned renderer for exact, validated APS descriptors. */ + apsNativeRenderer?: ApsNativeRendererHook; /** Initialises GPT slots with server-side bid targeting and calls refresh(). */ adInit?: () => void; /** GPT slot objects TS defined — used to destroy stale slots on SPA navigation. */ diff --git a/crates/trusted-server-js/lib/src/integrations/aps/render.ts b/crates/trusted-server-js/lib/src/integrations/aps/render.ts index 85f17adf9..f5faa60a1 100644 --- a/crates/trusted-server-js/lib/src/integrations/aps/render.ts +++ b/crates/trusted-server-js/lib/src/integrations/aps/render.ts @@ -1,7 +1,14 @@ import { log } from '../../core/log'; -import type { ApsPrebidRendererEntry, ApsRendererV1, TsjsApi } from '../../core/types'; +import type { + ApsNativeRendererHook, + ApsPrebidRendererEntry, + ApsRendererV1, + TsjsApi, +} from '../../core/types'; export const APS_RENDERER_PATH = '/integrations/aps/renderer'; +export const APS_RENDERING_MODE_META_NAME = 'trusted-server-aps-rendering-mode'; +export const APS_NATIVE_RENDERER_ACK_TIMEOUT_MS = 10_000; export const APS_RENDERER_SANDBOX = 'allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation'; export const APS_UNIVERSAL_CREATIVE_RENDERER_VERSION = 4; @@ -38,6 +45,13 @@ type ValidatedRendererCacheEntry = { renderer: ApsRendererV1; }; const validatedRendererCache = new WeakMap(); +const nativeDispatches = new Map(); + +function releaseNativeDispatch(slotId: string, dispatch: symbol): boolean { + if (nativeDispatches.get(slotId) !== dispatch) return false; + nativeDispatches.delete(slotId); + return true; +} function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); @@ -277,6 +291,133 @@ export function consumeApsPrebidRenderer(adId: string, expected: ApsPrebidRender return true; } +/** Whether the server explicitly selected the opt-in publisher-native hook mode. */ +export function isPublisherNativeApsRendering(): boolean { + return ( + document.head.querySelector( + `meta[name="${APS_RENDERING_MODE_META_NAME}"][content="publisher_native"]` + ) !== null + ); +} + +export interface DispatchApsRenderingOptions { + slotId: string; + renderer: unknown; + /** Existing Trusted Server owner, invoked only in the default mode. */ + trustedServer: (renderer: ApsRendererV1) => boolean; +} + +/** + * Dispatch a validated APS descriptor to exactly one configured rendering owner. + * + * Publisher hooks own side-effect cancellation and render completion. Trusted Server + * ignores superseded hook acknowledgements and never falls back to its iframe. + */ +export function dispatchApsRendering({ + slotId, + renderer: input, + trustedServer, +}: DispatchApsRenderingOptions): boolean | Promise { + // Record every attempt before any early return so it supersedes an older + // pending native acknowledgement for the same slot. + const dispatch = Symbol(slotId); + nativeDispatches.set(slotId, dispatch); + + const renderer = validateApsRenderer(input); + if (!renderer) { + releaseNativeDispatch(slotId, dispatch); + log.warn('APS renderer: rejected descriptor'); + return false; + } + if (!isPublisherNativeApsRendering()) { + try { + return trustedServer(renderer); + } finally { + releaseNativeDispatch(slotId, dispatch); + } + } + + let hook: ApsNativeRendererHook | undefined; + let render: ApsNativeRendererHook['render'] | undefined; + try { + hook = window.tsjs?.apsNativeRenderer; + render = hook?.render; + } catch { + releaseNativeDispatch(slotId, dispatch); + log.warn('APS native renderer: publisher hook lookup threw'); + return Promise.resolve(false); + } + if (!hook || typeof render !== 'function') { + releaseNativeDispatch(slotId, dispatch); + log.warn('APS native renderer: publisher hook is unavailable'); + return Promise.resolve(false); + } + + let response: unknown; + try { + response = Reflect.apply(render, hook, [{ version: 1, slotId, renderer }]); + } catch { + releaseNativeDispatch(slotId, dispatch); + log.warn('APS native renderer: publisher hook threw'); + return Promise.resolve(false); + } + + if (nativeDispatches.get(slotId) !== dispatch) { + log.warn('APS native renderer: ignored stale acknowledgement'); + return Promise.resolve(false); + } + + return new Promise((resolve) => { + let settled = false; + const settle = (accepted: boolean, warning?: string): void => { + if (settled) return; + settled = true; + clearTimeout(timeout); + if (!releaseNativeDispatch(slotId, dispatch)) { + log.warn('APS native renderer: ignored stale acknowledgement'); + resolve(false); + return; + } + if (warning) log.warn(warning); + resolve(accepted); + }; + const timeout = setTimeout(() => { + settle(false, 'APS native renderer: publisher hook acknowledgement timed out'); + }, APS_NATIVE_RENDERER_ACK_TIMEOUT_MS); + + Promise.resolve(response).then( + (value) => { + let accepted: boolean; + try { + if ( + !isRecord(value) || + (!hasExactKeys(value, ['accepted']) && !hasExactKeys(value, ['accepted', 'reason'])) || + typeof value.accepted !== 'boolean' || + (Object.prototype.hasOwnProperty.call(value, 'reason') && + typeof value.reason !== 'string') + ) { + settle(false, 'APS native renderer: publisher hook returned malformed acknowledgement'); + return; + } + accepted = value.accepted; + } catch { + settle(false, 'APS native renderer: publisher hook returned malformed acknowledgement'); + return; + } + + if (!accepted) { + settle(false, 'APS native renderer: publisher hook declined descriptor'); + return; + } + settle(true); + }, + () => { + settle(false, 'APS native renderer: publisher hook rejected'); + } + ); + }); +} + function createNonce(): string | undefined { if (typeof crypto === 'undefined' || typeof crypto.getRandomValues !== 'function') return undefined; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index f0df35974..d5fb0a59f 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -11,6 +11,7 @@ import { APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, apsRendererUrl, + dispatchApsRendering, consumeApsPrebidRenderer, getApsPrebidRenderer, validateApsRenderer, @@ -1676,29 +1677,49 @@ export function installTsRenderBridge(): void { e.stopImmediatePropagation(); if (!messageSourceBelongsToAdUnit(e.source, prebidRendererEntry.adUnitCode)) return; const renderer = validateApsRenderer(prebidRendererEntry.renderer); - const rendererUrl = apsRendererUrl(); - if (!renderer || !rendererUrl) return; - if (!hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; + if (!renderer || !hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; if (!consumeApsPrebidRenderer(adId, prebidRendererEntry)) return; recordConsumedPrebidApsId(consumedPrebidApsIds, adId, prebidRendererEntry.expiresAt); - port.postMessage( - JSON.stringify({ - message: 'Prebid Response', - adId, - renderer: APS_UNIVERSAL_CREATIVE_RENDERER, - rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - rendererUrl, - apsRenderer: renderer, - width: renderer.width, - height: renderer.height, - }) - ); - - try { - prebidRendererEntry.markUsed(); - } catch (err) { - log.warn(`[tsjs-gpt] APS Prebid markUsed callback threw for '${adId}'`, err); + const markUsed = (): void => { + try { + prebidRendererEntry.markUsed(); + } catch (err) { + log.warn(`[tsjs-gpt] APS Prebid markUsed callback threw for '${adId}'`, err); + } + }; + const dispatched = dispatchApsRendering({ + slotId: prebidRendererEntry.adUnitCode, + renderer, + trustedServer: (validatedRenderer) => { + const rendererUrl = apsRendererUrl(); + if (!rendererUrl) return false; + try { + port.postMessage( + JSON.stringify({ + message: 'Prebid Response', + adId, + renderer: APS_UNIVERSAL_CREATIVE_RENDERER, + rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, + rendererUrl, + apsRenderer: validatedRenderer, + width: validatedRenderer.width, + height: validatedRenderer.height, + }) + ); + return true; + } catch (err) { + log.warn(`[tsjs-gpt] APS Prebid response post failed for '${adId}'`, err); + return false; + } + }, + }); + if (typeof dispatched === 'boolean') { + if (dispatched) markUsed(); + } else { + void dispatched.then((accepted) => { + if (accepted) markUsed(); + }); } return; } @@ -1727,19 +1748,34 @@ export function installTsRenderBridge(): void { e.stopImmediatePropagation(); if (consumedServerApsBySlot.get(slotId) === adId) return; const renderer = validateApsRenderer(matchedBid.renderer); - const rendererUrl = apsRendererUrl(); - if (!renderer || !rendererUrl) return; + if (!renderer) return; consumedServerApsBySlot.set(slotId, adId); - port.postMessage( - JSON.stringify({ - message: 'Prebid Response', - adId, - renderer: APS_UNIVERSAL_CREATIVE_RENDERER, - rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - rendererUrl, - apsRenderer: renderer, - width: renderer.width, - height: renderer.height, + void Promise.resolve( + dispatchApsRendering({ + slotId, + renderer, + trustedServer: (validatedRenderer) => { + const rendererUrl = apsRendererUrl(); + if (!rendererUrl) return false; + try { + port.postMessage( + JSON.stringify({ + message: 'Prebid Response', + adId, + renderer: APS_UNIVERSAL_CREATIVE_RENDERER, + rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, + rendererUrl, + apsRenderer: validatedRenderer, + width: validatedRenderer.width, + height: validatedRenderer.height, + }) + ); + return true; + } catch (err) { + log.warn(`[tsjs-gpt] APS server response post failed for '${slotId}'`, err); + return false; + } + }, }) ); return; diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index dc17c9e87..1f8e032ba 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { AdUnit } from '../../src/core/types'; +import { APS_RENDERING_MODE_META_NAME } from '../../src/integrations/aps/render'; import envelope from '../fixtures/aps-renderer-v1.json'; async function flushRequestAds(): Promise { @@ -137,6 +138,57 @@ describe('request.requestAds', () => { expect(document.querySelector('#slot1 span')).toBeNull(); }); + it('contract test: dispatches a direct APS bid to the native publisher hook without an iframe', async () => { + const apsBid = envelope.seatbid[0].bid[0]; + const renderer = { + type: 'aps' as const, + version: 1 as const, + accountId: 'example-account-id', + bidId: apsBid.id, + tagType: apsBid.ext.tagtype as 'iframe', + creativeUrl: apsBid.ext.creativeurl, + aaxResponse: btoa(JSON.stringify(envelope)), + width: apsBid.w, + height: apsBid.h, + }; + const render = vi.fn().mockResolvedValue({ accepted: true }); + window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; + const marker = document.createElement('meta'); + marker.name = APS_RENDERING_MODE_META_NAME; + marker.content = 'publisher_native'; + document.head.appendChild(marker); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({ + seatbid: [ + { + seat: 'aps', + bid: [{ impid: 'slot1', ext: { trusted_server: { renderer } } }], + }, + ], + }), + }); + + try { + const { addAdUnits } = await import('../../src/core/registry'); + const { requestAds } = await import('../../src/core/request'); + document.body.innerHTML = '
existing
'; + addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } }); + + requestAds(); + await flushRequestAds(); + await Promise.resolve(); + + expect(render).toHaveBeenCalledWith({ version: 1, slotId: 'slot1', renderer }); + expect(document.querySelector('#slot1 iframe')).toBeNull(); + expect(document.querySelector('#slot1 span')).not.toBeNull(); + } finally { + marker.remove(); + } + }); + it('does not mutate the slot for an invalid APS descriptor', async () => { globalThis.fetch = vi.fn().mockResolvedValue({ ok: true, diff --git a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts index eae60c90e..627ae1268 100644 --- a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts @@ -4,11 +4,14 @@ import envelope from '../../fixtures/aps-renderer-v1.json'; import type { ApsRendererV1 } from '../../../src/core/types'; import { log } from '../../../src/core/log'; import { + APS_NATIVE_RENDERER_ACK_TIMEOUT_MS, APS_RENDERER_PATH, APS_RENDERER_SANDBOX, + APS_RENDERING_MODE_META_NAME, APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, apsRendererUrl, + dispatchApsRendering, getApsPrebidRenderer, parseApsRendererDescriptor, registerApsPrebidRenderer, @@ -16,6 +19,19 @@ import { validateApsRenderer, } from '../../../src/integrations/aps/render'; +function enablePublisherNativeMode(): void { + const marker = document.createElement('meta'); + marker.name = APS_RENDERING_MODE_META_NAME; + marker.content = 'publisher_native'; + document.head.appendChild(marker); +} + +function disablePublisherNativeMode(): void { + document.head + .querySelectorAll(`meta[name="${APS_RENDERING_MODE_META_NAME}"]`) + .forEach((marker) => marker.remove()); +} + function encodeBytes(bytes: Uint8Array): string { let binary = ''; for (const byte of bytes) binary += String.fromCharCode(byte); @@ -261,6 +277,256 @@ describe('Prebid APS renderer registry', () => { }); }); +describe('publisher-native APS hook contract tests', () => { + beforeEach(() => { + document.body.innerHTML = '
existing
'; + enablePublisherNativeMode(); + }); + + afterEach(() => { + disablePublisherNativeMode(); + delete window.tsjs; + vi.restoreAllMocks(); + document.body.innerHTML = ''; + }); + + it('contract test: sends the exact frozen descriptor to an accepting publisher hook without an iframe', async () => { + const render = vi.fn().mockResolvedValue({ accepted: true }); + window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; + const unrelatedMarker = document.createElement('meta'); + unrelatedMarker.name = APS_RENDERING_MODE_META_NAME; + unrelatedMarker.content = 'trusted_server'; + document.head.appendChild(unrelatedMarker); + + const accepted = await dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => { + throw new Error('trusted renderer must not run'); + }, + }); + + expect(accepted).toBe(true); + expect(render).toHaveBeenCalledTimes(1); + expect(render).toHaveBeenCalledWith({ + version: 1, + slotId: 'fictional-slot', + renderer: descriptor(), + }); + const payload = render.mock.calls[0][0]; + expect(Object.keys(payload).sort()).toEqual(['renderer', 'slotId', 'version']); + expect(Object.isFrozen(payload.renderer)).toBe(true); + expect(document.querySelector('iframe')).toBeNull(); + }); + + it('contract test: declines missing, throwing, rejecting, and malformed hooks without fallback', async () => { + const trustedServer = vi.fn(() => true); + await expect( + dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) + ).resolves.toBe(false); + + window.tsjs = { + apsNativeRenderer: { + render: () => { + throw new Error('fictional'); + }, + }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) + ).resolves.toBe(false); + + window.tsjs = { + apsNativeRenderer: { render: () => Promise.reject(new Error('fictional')) }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) + ).resolves.toBe(false); + + window.tsjs = { + apsNativeRenderer: { render: () => ({ accepted: false }) }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) + ).resolves.toBe(false); + + window.tsjs = { + apsNativeRenderer: { render: () => ({ accepted: 'yes' }) }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) + ).resolves.toBe(false); + + expect(trustedServer).not.toHaveBeenCalled(); + expect(document.querySelector('iframe')).toBeNull(); + }); + + it('contract test: ignores a stale acknowledgement after a replacement dispatch', async () => { + let resolveFirst: ((value: { accepted: boolean }) => void) | undefined; + const render = vi + .fn() + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFirst = resolve; + }) + ) + .mockResolvedValueOnce({ accepted: true }); + window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; + + const first = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + const second = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + resolveFirst!({ accepted: true }); + + await expect(first).resolves.toBe(false); + await expect(second).resolves.toBe(true); + }); + + it('contract test: a missing hook supersedes an older pending dispatch', async () => { + let resolveFirst: ((value: { accepted: boolean }) => void) | undefined; + const render = vi.fn( + () => + new Promise<{ accepted: boolean }>((resolve) => { + resolveFirst = resolve; + }) + ); + window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; + + const first = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + delete window.tsjs!.apsNativeRenderer; + const second = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + resolveFirst!({ accepted: true }); + + await expect(second).resolves.toBe(false); + await expect(first).resolves.toBe(false); + }); + + it('contract test: a trusted-server dispatch supersedes an older native dispatch', async () => { + let resolveFirst: ((value: { accepted: boolean }) => void) | undefined; + window.tsjs = { + apsNativeRenderer: { + render: () => + new Promise<{ accepted: boolean }>((resolve) => { + resolveFirst = resolve; + }), + }, + } as typeof window.tsjs; + const first = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + + disablePublisherNativeMode(); + const trustedServer = vi.fn(() => true); + const second = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer, + }); + resolveFirst!({ accepted: true }); + + expect(second).toBe(true); + expect(trustedServer).toHaveBeenCalledOnce(); + await expect(first).resolves.toBe(false); + }); + + it('contract test: contains throwing hook and acknowledgement accessors', async () => { + const tsjs = {} as NonNullable; + Object.defineProperty(tsjs, 'apsNativeRenderer', { + get: () => { + throw new Error('fictional hook lookup failure'); + }, + }); + window.tsjs = tsjs; + + await expect( + dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }) + ).resolves.toBe(false); + + window.tsjs = { + apsNativeRenderer: { + render: () => + new Proxy( + { accepted: true }, + { + ownKeys: () => { + throw new Error('fictional acknowledgement inspection failure'); + }, + } + ), + }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }) + ).resolves.toBe(false); + }); + + it('contract test: times out a hook and ignores its late acknowledgement', async () => { + vi.useFakeTimers(); + try { + let resolveHook: ((value: { accepted: boolean }) => void) | undefined; + window.tsjs = { + apsNativeRenderer: { + render: () => + new Promise<{ accepted: boolean }>((resolve) => { + resolveHook = resolve; + }), + }, + } as typeof window.tsjs; + + const result = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + await vi.advanceTimersByTimeAsync(APS_NATIVE_RENDERER_ACK_TIMEOUT_MS); + + await expect(result).resolves.toBe(false); + expect(vi.getTimerCount()).toBe(0); + resolveHook!({ accepted: true }); + await Promise.resolve(); + + window.tsjs = { + apsNativeRenderer: { render: () => ({ accepted: true }) }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }) + ).resolves.toBe(true); + } finally { + vi.useRealTimers(); + } + }); +}); + describe('direct APS rendering', () => { beforeEach(() => { document.body.innerHTML = '
existing
'; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 179a810d5..7f23cd394 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -7,6 +7,15 @@ import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vites import envelope from '../../fixtures/aps-renderer-v1.json'; import type { AuctionBidData, TsjsApi } from '../../../src/core/types'; +import { APS_RENDERING_MODE_META_NAME } from '../../../src/integrations/aps/render'; + +function enablePublisherNativeMode(): HTMLMetaElement { + const marker = document.createElement('meta'); + marker.name = APS_RENDERING_MODE_META_NAME; + marker.content = 'publisher_native'; + document.head.appendChild(marker); + return marker; +} function apsRenderer() { const bid = envelope.seatbid[0].bid[0]; @@ -3197,6 +3206,79 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('contract test: delegates a server APS owner to the native hook without a Universal Creative response', async () => { + const renderer = apsRenderer(); + const render = vi.fn().mockResolvedValue({ accepted: true }); + (window as TestWindow).tsjs.bids.homepage_header = { + hb_adid: renderer.bidId, + renderer, + }; + (window as TestWindow).tsjs.apsNativeRenderer = { render }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: renderer.bidId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + bridgeListener(request); + await Promise.resolve(); + await Promise.resolve(); + + expect(render).toHaveBeenCalledOnce(); + expect(render).toHaveBeenCalledWith({ + version: 1, + slotId: 'homepage_header', + renderer, + }); + expect(portMessages).toEqual([]); + expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); + } finally { + marker.remove(); + } + }); + + it('contract test: declines a server APS owner without a Universal Creative response or fallback', async () => { + const renderer = apsRenderer(); + const render = vi.fn().mockResolvedValue({ accepted: false }); + (window as TestWindow).tsjs.bids.homepage_header = { + hb_adid: renderer.bidId, + renderer, + }; + (window as TestWindow).tsjs.apsNativeRenderer = { render }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: renderer.bidId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + await Promise.resolve(); + await Promise.resolve(); + bridgeListener(request); + + expect(render).toHaveBeenCalledOnce(); + expect(portMessages).toEqual([]); + expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); + } finally { + marker.remove(); + } + }); + it('serves a registered Prebid APS renderer when its generated ad ID differs from the APS bid ID', async () => { const renderer = apsRenderer(); const prebidAdId = 'prebid-generated-ad-id'; @@ -3252,6 +3334,92 @@ describe('installTsRenderBridge', () => { foreignIframe.remove(); }); + it('contract test: declines a registered APS capability without a Universal Creative response or markUsed', async () => { + const renderer = apsRenderer(); + const prebidAdId = 'native-prebid-decline-ad-id'; + const markUsed = vi.fn(); + const render = vi.fn().mockResolvedValue({ accepted: false }); + (window as TestWindow).tsjs.apsPrebidRenderers = { + [prebidAdId]: { + adUnitCode: 'div-header', + renderer, + registeredAt: Date.now(), + expiresAt: Date.now() + 60_000, + markUsed, + }, + }; + (window as TestWindow).tsjs.apsNativeRenderer = { render }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: prebidAdId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + await Promise.resolve(); + await Promise.resolve(); + bridgeListener(request); + + expect(render).toHaveBeenCalledOnce(); + expect(markUsed).not.toHaveBeenCalled(); + expect(portMessages).toEqual([]); + expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); + expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); + } finally { + marker.remove(); + } + }); + + it('contract test: consumes a registered APS capability and marks it used only after native acceptance', async () => { + const renderer = apsRenderer(); + const prebidAdId = 'native-prebid-ad-id'; + const markUsed = vi.fn(); + const render = vi.fn().mockResolvedValue({ accepted: true }); + (window as TestWindow).tsjs.apsPrebidRenderers = { + [prebidAdId]: { + adUnitCode: 'div-header', + renderer, + registeredAt: Date.now(), + expiresAt: Date.now() + 60_000, + markUsed, + }, + }; + (window as TestWindow).tsjs.apsNativeRenderer = { render }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: prebidAdId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + expect(markUsed).not.toHaveBeenCalled(); + await Promise.resolve(); + await Promise.resolve(); + bridgeListener(request); + + expect(render).toHaveBeenCalledOnce(); + expect(markUsed).toHaveBeenCalledOnce(); + expect(portMessages).toEqual([]); + expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); + } finally { + marker.remove(); + } + }); + it('still serves the APS renderer when markUsed throws', async () => { const renderer = apsRenderer(); const prebidAdId = 'throwing-mark-used-ad-id'; diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index 658f5d5bc..9a5f4cb7e 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -36,6 +36,8 @@ debug = false # inventory_domain = "publisher.example" # inventory_page_origin = "https://www.publisher.example" allow_script_creatives = false +# Default. Set publisher_native only with a publisher-installed hook (below). +rendering_mode = "trusted_server" [auction] enabled = true @@ -49,6 +51,31 @@ timeout_ms = 2000 `allow_script_creatives` defaults to `false`. While disabled, APS script bids are rejected before per-impression reduction, floors, mediation, and winner selection. Enable it only for a controlled cohort after the browser-security checks in [Rollout](#rollout) pass. +`rendering_mode` is a strict enum: `trusted_server` (the default) retains the opaque static renderer route, and `publisher_native` disables that route and emits the inert `` head marker selecting the publisher hook below. The marker works under a publisher CSP that blocks inline scripts. Unknown values fail configuration deserialization. + +### Publisher-native hook experiment + +`publisher_native` is an opt-in publisher integration seam, **not** APS compatibility proof. No public APS API was found that accepts an externally selected OpenRTB `aaxResponse` for native rendering. In particular, `apstag.setDisplayBids()` operates on APS's own `fetchBids()` state and is not an ingestion API for the exact Trusted Server-selected bid. Trusted Server does not call `apstag`, `fetchBids`, or `setDisplayBids`, mutate APS internals, or start a second auction. + +Before Trusted Server JS receives a selected descriptor, the publisher must install this versioned hook: + +```js +window.tsjs = window.tsjs || {} +window.tsjs.apsNativeRenderer = { + render({ version, slotId, renderer }) { + // version is exactly 1; renderer is frozen and fully validated. + // Render only this exact selected descriptor through publisher-owned logic. + return { accepted: true } + }, +} +``` + +The hook receives exactly `{ version: 1, slotId, renderer }`. It must return or resolve an object with `accepted: boolean`; `reason?: string` is allowed for fictional-safe diagnostics. Missing hooks, throws, rejected promises, malformed acknowledgements, `{ accepted: false }`, and acknowledgements that take longer than 10 seconds visibly decline the bid. They never fall back to the Trusted Server iframe or send a Universal Creative renderer response. A newer dispatch makes an older acknowledgement stale and ignored; late acknowledgements after the timeout are also ignored. Hook implementations own cancellation of any already-started side effect and actual render completion. + +For a client-side Prebid APS capability, Trusted Server consumes the one-shot capability before delegation and calls `markWinningBidAsUsed` only after `accepted: true`. For server/GPT ownership, it similarly claims the slot/ad ID before invoking the hook. This prevents native and Trusted Server rendering from both owning the same response, but an accepting hook is responsible for real rendering semantics. + +Disable or coordinate publisher-native APS demand for every `publisher_native` cohort. Otherwise native APS demand and this server-selected bid can duplicate demand. Validate a controlled real publisher/account setup with the APS account team before any production rollout. + Set `inventory_domain` and `inventory_page_origin` together only when the public deployment hostname differs from the inventory identity authorized by APS. The domain becomes `site.domain`. The HTTPS page origin replaces the current page's scheme and host while preserving its path; query and fragment data are removed before forwarding. The origin must be the inventory domain or one of its subdomains and cannot include credentials, a port, path, query, or fragment. These values come only from operator configuration; Trusted Server never accepts APS inventory identity from the client auction payload. APS uses ordinary auction slot IDs and banner formats. Legacy creative-opportunity APS `slot_id` configuration is accepted for compatibility but ignored, and `bidders.aps.slotID` is not required. Remove both during migration. @@ -150,7 +177,7 @@ Trusted Server does not insert APS creative markup into the publisher document. Seats, `impid`, markup, notifications, user-sync data, sibling bids, losing seats, and unknown fields are not exposed. The browser decodes this envelope and cross-checks the ID, dimensions, URL, and tag type before any DOM mutation or message suppression. -Both rendering paths use `GET /integrations/aps/renderer`, a static Trusted Server document with its own restrictive CSP. The document initializes the account-keyed APS queue and then loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. +In `trusted_server` mode, both rendering paths use `GET /integrations/aps/renderer`, a static Trusted Server document with its own restrictive CSP. The document initializes the account-keyed APS queue and then loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. The outer iframe uses these sandbox permissions: @@ -167,11 +194,11 @@ It deliberately omits `allow-same-origin`, so APS and bidder execution remains b ### Direct `/auction` -The TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. +In `trusted_server` mode, the TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. In `publisher_native` mode it instead calls the explicit publisher hook and creates no Trusted Server iframe. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. ### GAM and Universal Creative -For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program that creates the same opaque renderer iframe. +For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. In `publisher_native` mode the ownership-checked bridge calls the publisher hook without sending a Universal Creative renderer response; in `trusted_server` mode it uses the static dynamic renderer described below. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program that creates the same opaque renderer iframe. For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes the capability once, and passes the APS bid ID separately to the Amazon runner. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 19ecda4a5..6e844e385 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -155,6 +155,9 @@ debug = false # inventory_page_origin = "https://www.publisher.example" # Script creatives require separate security validation before opt-in. allow_script_creatives = false +# Default: Trusted Server's opaque static renderer route. Set publisher_native only +# when the publisher installs the documented tsjs.apsNativeRenderer hook. +rendering_mode = "trusted_server" [integrations.google_tag_manager] enabled = false From 6078a46c34c33bed5f727789dc2c8d45691ef647 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 19 Aug 2026 11:38:41 -0500 Subject: [PATCH 374/395] Inject APS publisher-native runner The test publisher cannot install a custom rendering hook. Reuse the existing APS Prebid creative runner contract in a publisher-origin friendly frame so the experiment remains self-contained while preserving the selected server bid and avoiding a second auction.\n\nDocument the larger security surface and retain the opaque renderer as the default.\n\nSee also: #999 --- .../src/integrations/aps.rs | 4 +- .../browser/tests/shared/aps-renderer.spec.ts | 111 +++++++- .../trusted-server-js/lib/src/core/types.ts | 17 -- .../lib/src/integrations/aps/render.ts | 211 +++++++++++----- .../lib/test/core/request.test.ts | 33 ++- .../lib/test/integrations/aps/render.test.ts | 239 +++++++----------- .../lib/test/integrations/gpt/ad_init.test.ts | 65 +++-- docs/guide/integrations/aps.md | 54 ++-- trusted-server.example.toml | 2 +- 9 files changed, 441 insertions(+), 295 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index fba758973..d6037ce5d 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -121,7 +121,7 @@ pub enum ApsRenderingMode { /// Render through Trusted Server's opaque static renderer route. #[default] TrustedServer, - /// Delegate rendering to the publisher's explicit browser hook. + /// Render through the injected APS runner in a publisher-origin friendly frame. PublisherNative, } @@ -2433,7 +2433,7 @@ mod tests { } #[test] - fn publisher_native_config_registers_hook_mode_without_renderer_route() { + fn publisher_native_config_registers_runner_mode_without_renderer_route() { let mut settings = create_test_settings(); settings .integrations diff --git a/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts b/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts index 6c0301e2c..2e8e418d2 100644 --- a/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts +++ b/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts @@ -192,7 +192,7 @@ const SCRIPT_CREATIVE = `(function(){ }, '*'); })();`; -test.describe("APS opaque renderer", () => { +test.describe("APS rendering", () => { test("renders a trustedServer adapter bid using Prebid's generated GAM ad ID", async ({ page, }) => { @@ -956,4 +956,113 @@ parent.postMessage(JSON.stringify({ 1, ); }); + + test("renders publisher-native mode through the injected friendly-frame runner", async ({ + page, + }) => { + const publisherOrigin = "https://publisher.example"; + const auctionUrl = `${publisherOrigin}/auction`; + const testUrl = `${publisherOrigin}/aps-publisher-native-test`; + const renderer = descriptor("iframe"); + let runnerRequests = 0; + + await page.route(RUNNER_URL, async (route) => { + runnerRequests += 1; + await route.fulfill({ + status: 200, + contentType: "application/javascript", + body: FAKE_RUNNER, + }); + }); + await page.route(IFRAME_CREATIVE_URL, async (route) => { + await route.fulfill({ + status: 200, + contentType: "text/html", + body: IFRAME_CREATIVE, + }); + }); + await page.route(auctionUrl, async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + id: "fictional-native-auction", + seatbid: [ + { + seat: "aps", + bid: [ + { + id: renderer.bidId, + impid: "publisher-native-slot", + price: 1.23, + w: renderer.width, + h: renderer.height, + ext: { trusted_server: { renderer } }, + }, + ], + }, + ], + ext: {}, + }), + }); + }); + await page.route(testUrl, async (route) => { + await route.fulfill({ + status: 200, + contentType: "text/html", + headers: { + "Content-Security-Policy": + "default-src 'none'; script-src 'unsafe-inline' https://client.aps.amazon-adsystem.com https://creative.example; connect-src 'self'; frame-src https://creative.example", + }, + body: ` + +
existing publisher content
`, + }); + }); + + await page.goto(testUrl); + await page.addScriptTag({ path: clientAuctionBundlePaths().core }); + await page.evaluate(() => { + const tsjs = ( + window as unknown as { + tsjs: { + addAdUnits(units: Array>): void; + requestAds(): void; + }; + } + ).tsjs; + tsjs.addAdUnits([ + { + code: "publisher-native-slot", + mediaTypes: { banner: { sizes: [[300, 250]] } }, + bids: [], + }, + ]); + tsjs.requestAds(); + }); + + await expect.poll(() => runnerRequests).toBe(1); + const frame = page.locator("#publisher-native-slot > iframe"); + await expect(frame).toHaveCount(1); + await expect(frame).toBeVisible(); + expect(await frame.getAttribute("sandbox")).toBeNull(); + await expect( + frame + .contentFrame() + .locator(`iframe[src="${IFRAME_CREATIVE_URL}"]`), + ).toHaveCount(1); + await expect( + page.locator("#publisher-native-slot .existing"), + ).toHaveCount(0); + expect( + await page + .locator("#publisher-native-slot") + .evaluate( + (slot) => + slot.querySelectorAll( + 'iframe[src*="/integrations/aps/renderer"]', + ).length, + ), + ).toBe(0); + }); }); diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 192b82033..0c68d43fe 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -68,21 +68,6 @@ export interface ApsRendererV1 { export type AuctionBidRenderer = ApsRendererV1; -/** Explicit acknowledgement returned by the opt-in publisher-native APS hook. */ -export interface ApsNativeRendererResult { - accepted: boolean; - reason?: string; -} - -/** Publisher-owned rendering seam for a fully validated APS descriptor. */ -export interface ApsNativeRendererHook { - render(input: { - version: 1; - slotId: string; - renderer: ApsRendererV1; - }): ApsNativeRendererResult | Promise; -} - /** A client-side Prebid bid's generated ad ID bound to its APS render capability. */ export interface ApsPrebidRendererEntry { adUnitCode: string; @@ -403,8 +388,6 @@ export interface TsjsApi { * `hb_adid`. The Universal Creative bridge consumes each entry at most once. */ apsPrebidRenderers?: Record; - /** Opt-in publisher-owned renderer for exact, validated APS descriptors. */ - apsNativeRenderer?: ApsNativeRendererHook; /** Initialises GPT slots with server-side bid targeting and calls refresh(). */ adInit?: () => void; /** GPT slot objects TS defined — used to destroy stale slots on SPA navigation. */ diff --git a/crates/trusted-server-js/lib/src/integrations/aps/render.ts b/crates/trusted-server-js/lib/src/integrations/aps/render.ts index f5faa60a1..25bf82231 100644 --- a/crates/trusted-server-js/lib/src/integrations/aps/render.ts +++ b/crates/trusted-server-js/lib/src/integrations/aps/render.ts @@ -1,14 +1,12 @@ import { log } from '../../core/log'; -import type { - ApsNativeRendererHook, - ApsPrebidRendererEntry, - ApsRendererV1, - TsjsApi, -} from '../../core/types'; +import { findSlot } from '../../core/render'; +import type { ApsPrebidRendererEntry, ApsRendererV1, TsjsApi } from '../../core/types'; export const APS_RENDERER_PATH = '/integrations/aps/renderer'; export const APS_RENDERING_MODE_META_NAME = 'trusted-server-aps-rendering-mode'; -export const APS_NATIVE_RENDERER_ACK_TIMEOUT_MS = 10_000; +export const APS_PREBID_CREATIVE_RUNNER_URL = + 'https://client.aps.amazon-adsystem.com/prebid-creative.js'; +export const APS_NATIVE_RENDERER_TIMEOUT_MS = 10_000; export const APS_RENDERER_SANDBOX = 'allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation'; export const APS_UNIVERSAL_CREATIVE_RENDERER_VERSION = 4; @@ -53,6 +51,29 @@ function releaseNativeDispatch(slotId: string, dispatch: symbol): boolean { return true; } +function findApsContainer(slotId: string): HTMLElement | null { + const direct = findSlot(slotId); + if (direct) return direct; + + try { + for (const [divId, mappedSlotId] of Object.entries(window.tsjs?.divToSlotId ?? {})) { + if (mappedSlotId !== slotId) continue; + const mapped = findSlot(divId); + if (mapped) return mapped; + } + + const configuredDivId = window.tsjs?.adSlots?.find((slot) => slot.id === slotId)?.div_id; + return configuredDivId ? findSlot(configuredDivId) : null; + } catch { + return null; + } +} + +function cancelPendingApsRendering(slotId: string): void { + const container = findApsContainer(slotId); + if (container) pendingFrameCancels.get(container)?.(); +} + function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } @@ -291,7 +312,7 @@ export function consumeApsPrebidRenderer(adId: string, expected: ApsPrebidRender return true; } -/** Whether the server explicitly selected the opt-in publisher-native hook mode. */ +/** Whether the server explicitly selected the opt-in publisher-native runner mode. */ export function isPublisherNativeApsRendering(): boolean { return ( document.head.querySelector( @@ -310,16 +331,17 @@ export interface DispatchApsRenderingOptions { /** * Dispatch a validated APS descriptor to exactly one configured rendering owner. * - * Publisher hooks own side-effect cancellation and render completion. Trusted Server - * ignores superseded hook acknowledgements and never falls back to its iframe. + * Native mode loads APS's fixed Prebid creative runner in a publisher-origin friendly + * frame. Superseded attempts are cancelled and never fall back to the opaque renderer. */ export function dispatchApsRendering({ slotId, renderer: input, trustedServer, }: DispatchApsRenderingOptions): boolean | Promise { - // Record every attempt before any early return so it supersedes an older - // pending native acknowledgement for the same slot. + // Every attempt supersedes a pending frame for this slot, including an invalid + // replacement that fails before a new frame can be created. + cancelPendingApsRendering(slotId); const dispatch = Symbol(slotId); nativeDispatches.set(slotId, dispatch); @@ -337,84 +359,133 @@ export function dispatchApsRendering({ } } - let hook: ApsNativeRendererHook | undefined; - let render: ApsNativeRendererHook['render'] | undefined; + let rendering: Promise; try { - hook = window.tsjs?.apsNativeRenderer; - render = hook?.render; + rendering = renderApsPublisherNative({ slotId, renderer }); } catch { releaseNativeDispatch(slotId, dispatch); - log.warn('APS native renderer: publisher hook lookup threw'); - return Promise.resolve(false); - } - if (!hook || typeof render !== 'function') { - releaseNativeDispatch(slotId, dispatch); - log.warn('APS native renderer: publisher hook is unavailable'); + log.warn('APS native renderer: failed to start publisher-origin frame'); return Promise.resolve(false); } - let response: unknown; - try { - response = Reflect.apply(render, hook, [{ version: 1, slotId, renderer }]); - } catch { - releaseNativeDispatch(slotId, dispatch); - log.warn('APS native renderer: publisher hook threw'); - return Promise.resolve(false); - } + return rendering.then((accepted) => { + if (!releaseNativeDispatch(slotId, dispatch)) { + log.warn('APS native renderer: ignored stale completion'); + return false; + } + return accepted; + }); +} + +export interface RenderApsPublisherNativeOptions { + slotId: string; + renderer: unknown; +} - if (nativeDispatches.get(slotId) !== dispatch) { - log.warn('APS native renderer: ignored stale acknowledgement'); +/** Render the exact selected response through APS's fixed runner in a friendly iframe. */ +export function renderApsPublisherNative({ + slotId, + renderer: input, +}: RenderApsPublisherNativeOptions): Promise { + const renderer = validateApsRenderer(input); + const container = findApsContainer(slotId); + if (!renderer || !container) { + log.warn( + renderer ? 'APS native renderer: slot not found' : 'APS renderer: rejected descriptor' + ); return Promise.resolve(false); } + // Keep an already committed creative visible until the replacement runner loads. + pendingFrameCancels.get(container)?.(); + const iframe = document.createElement('iframe'); + iframe.title = 'Ad content'; + iframe.width = String(renderer.width); + iframe.height = String(renderer.height); + iframe.style.border = '0'; + iframe.style.display = 'none'; + activeFrames.set(container, iframe); + return new Promise((resolve) => { let settled = false; - const settle = (accepted: boolean, warning?: string): void => { + let runner: HTMLScriptElement | undefined; + + const cleanup = (): void => { + window.clearTimeout(timeoutId); + runner?.removeEventListener('load', commit); + runner?.removeEventListener('error', fail); + }; + const finish = (accepted: boolean, warning?: string): void => { if (settled) return; settled = true; - clearTimeout(timeout); - if (!releaseNativeDispatch(slotId, dispatch)) { - log.warn('APS native renderer: ignored stale acknowledgement'); + cleanup(); + if (pendingFrameCancels.get(container) === cancel) pendingFrameCancels.delete(container); + + if (!accepted || activeFrames.get(container) !== iframe || !iframe.isConnected) { + if (activeFrames.get(container) === iframe) activeFrames.delete(container); + iframe.remove(); + if (warning) log.warn(warning); resolve(false); return; } - if (warning) log.warn(warning); - resolve(accepted); - }; - const timeout = setTimeout(() => { - settle(false, 'APS native renderer: publisher hook acknowledgement timed out'); - }, APS_NATIVE_RENDERER_ACK_TIMEOUT_MS); - - Promise.resolve(response).then( - (value) => { - let accepted: boolean; - try { - if ( - !isRecord(value) || - (!hasExactKeys(value, ['accepted']) && !hasExactKeys(value, ['accepted', 'reason'])) || - typeof value.accepted !== 'boolean' || - (Object.prototype.hasOwnProperty.call(value, 'reason') && - typeof value.reason !== 'string') - ) { - settle(false, 'APS native renderer: publisher hook returned malformed acknowledgement'); - return; - } - accepted = value.accepted; - } catch { - settle(false, 'APS native renderer: publisher hook returned malformed acknowledgement'); - return; - } - if (!accepted) { - settle(false, 'APS native renderer: publisher hook declined descriptor'); - return; - } - settle(true); - }, - () => { - settle(false, 'APS native renderer: publisher hook rejected'); + for (const child of Array.from(container.children)) { + if (child !== iframe) child.remove(); } + iframe.style.display = ''; + resolve(true); + }; + const cancel = (): void => finish(false); + function fail(): void { + finish(false, 'APS native renderer: creative runner failed'); + } + function commit(): void { + finish(true); + } + + const timeoutId = window.setTimeout( + () => finish(false, 'APS native renderer: creative runner timed out'), + APS_NATIVE_RENDERER_TIMEOUT_MS ); + pendingFrameCancels.set(container, cancel); + container.appendChild(iframe); + + try { + const frameWindow = iframe.contentWindow as + | (Window & + typeof globalThis & { + _aps: Map> }>; + }) + | null; + const frameDocument = iframe.contentDocument; + if (!frameWindow || !frameDocument) { + fail(); + return; + } + + frameDocument.open(); + frameDocument.write( + '' + ); + frameDocument.close(); + frameWindow._aps = new Map(); + frameWindow._aps.set(renderer.accountId, { + queue: [ + new frameWindow.CustomEvent('prebid/creative/render', { + detail: { aaxResponse: renderer.aaxResponse, seatBidId: renderer.bidId }, + }), + ], + store: new Map([['listeners', new Map()]]), + }); + + runner = frameDocument.createElement('script'); + runner.src = APS_PREBID_CREATIVE_RUNNER_URL; + runner.addEventListener('load', commit, { once: true }); + runner.addEventListener('error', fail, { once: true }); + frameDocument.head.appendChild(runner); + } catch { + fail(); + } }); } diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index 1f8e032ba..bfc6c88f6 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -1,7 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { AdUnit } from '../../src/core/types'; -import { APS_RENDERING_MODE_META_NAME } from '../../src/integrations/aps/render'; +import { + APS_PREBID_CREATIVE_RUNNER_URL, + APS_RENDERING_MODE_META_NAME, +} from '../../src/integrations/aps/render'; import envelope from '../fixtures/aps-renderer-v1.json'; async function flushRequestAds(): Promise { @@ -138,7 +141,7 @@ describe('request.requestAds', () => { expect(document.querySelector('#slot1 span')).toBeNull(); }); - it('contract test: dispatches a direct APS bid to the native publisher hook without an iframe', async () => { + it('contract test: renders a direct APS bid through the injected native runner', async () => { const apsBid = envelope.seatbid[0].bid[0]; const renderer = { type: 'aps' as const, @@ -151,8 +154,6 @@ describe('request.requestAds', () => { width: apsBid.w, height: apsBid.h, }; - const render = vi.fn().mockResolvedValue({ accepted: true }); - window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; const marker = document.createElement('meta'); marker.name = APS_RENDERING_MODE_META_NAME; marker.content = 'publisher_native'; @@ -179,11 +180,27 @@ describe('request.requestAds', () => { requestAds(); await flushRequestAds(); - await Promise.resolve(); - - expect(render).toHaveBeenCalledWith({ version: 1, slotId: 'slot1', renderer }); - expect(document.querySelector('#slot1 iframe')).toBeNull(); + const frame = document.querySelector('#slot1 iframe')!; + const runner = frame.contentDocument?.querySelector('script'); + expect(runner).not.toBeNull(); + const frameWindow = frame.contentWindow as unknown as { + _aps: Map>> }>; + }; + const queued = frameWindow._aps.get(renderer.accountId)?.queue[0]; + + expect(frame.getAttribute('sandbox')).toBeNull(); + expect(runner!.src).toBe(APS_PREBID_CREATIVE_RUNNER_URL); + expect(queued?.type).toBe('prebid/creative/render'); + expect(queued?.detail).toEqual({ + aaxResponse: renderer.aaxResponse, + seatBidId: renderer.bidId, + }); expect(document.querySelector('#slot1 span')).not.toBeNull(); + + runner!.dispatchEvent(new Event('load')); + await Promise.resolve(); + expect(document.querySelector('#slot1 span')).toBeNull(); + expect(frame.style.display).toBe(''); } finally { marker.remove(); } diff --git a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts index 627ae1268..dfe673ac5 100644 --- a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts @@ -4,7 +4,8 @@ import envelope from '../../fixtures/aps-renderer-v1.json'; import type { ApsRendererV1 } from '../../../src/core/types'; import { log } from '../../../src/core/log'; import { - APS_NATIVE_RENDERER_ACK_TIMEOUT_MS, + APS_NATIVE_RENDERER_TIMEOUT_MS, + APS_PREBID_CREATIVE_RUNNER_URL, APS_RENDERER_PATH, APS_RENDERER_SANDBOX, APS_RENDERING_MODE_META_NAME, @@ -32,6 +33,20 @@ function disablePublisherNativeMode(): void { .forEach((marker) => marker.remove()); } +function nativeRunnerState(frame: HTMLIFrameElement): { + runner: HTMLScriptElement; + event: CustomEvent<{ aaxResponse: string; seatBidId: string }>; +} { + const runner = frame.contentDocument?.querySelector('script'); + const frameWindow = frame.contentWindow as unknown as { + _aps: Map> }>; + }; + const account = frameWindow._aps.get('example-account-id'); + expect(runner).not.toBeNull(); + expect(account?.queue).toHaveLength(1); + return { runner: runner!, event: account!.queue[0] }; +} + function encodeBytes(bytes: Uint8Array): string { let binary = ''; for (const byte of bytes) binary += String.fromCharCode(byte); @@ -277,7 +292,7 @@ describe('Prebid APS renderer registry', () => { }); }); -describe('publisher-native APS hook contract tests', () => { +describe('publisher-native APS runner contract tests', () => { beforeEach(() => { document.body.innerHTML = '
existing
'; enablePublisherNativeMode(); @@ -290,143 +305,98 @@ describe('publisher-native APS hook contract tests', () => { document.body.innerHTML = ''; }); - it('contract test: sends the exact frozen descriptor to an accepting publisher hook without an iframe', async () => { - const render = vi.fn().mockResolvedValue({ accepted: true }); - window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; + it('queues the exact selected response for the fixed APS runner and commits on load', async () => { + const trustedServer = vi.fn(() => true); const unrelatedMarker = document.createElement('meta'); unrelatedMarker.name = APS_RENDERING_MODE_META_NAME; unrelatedMarker.content = 'trusted_server'; document.head.appendChild(unrelatedMarker); - const accepted = await dispatchApsRendering({ + const accepted = dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), - trustedServer: () => { - throw new Error('trusted renderer must not run'); - }, + trustedServer, }); - - expect(accepted).toBe(true); - expect(render).toHaveBeenCalledTimes(1); - expect(render).toHaveBeenCalledWith({ - version: 1, - slotId: 'fictional-slot', - renderer: descriptor(), + const slot = document.getElementById('fictional-slot')!; + const frame = slot.querySelector('iframe')!; + const { runner, event } = nativeRunnerState(frame); + + expect(frame.getAttribute('sandbox')).toBeNull(); + expect(frame.style.display).toBe('none'); + expect(runner.src).toBe(APS_PREBID_CREATIVE_RUNNER_URL); + expect(event.type).toBe('prebid/creative/render'); + expect(event.detail).toEqual({ + aaxResponse: descriptor().aaxResponse, + seatBidId: descriptor().bidId, }); - const payload = render.mock.calls[0][0]; - expect(Object.keys(payload).sort()).toEqual(['renderer', 'slotId', 'version']); - expect(Object.isFrozen(payload.renderer)).toBe(true); - expect(document.querySelector('iframe')).toBeNull(); + expect(slot.querySelector('span')).not.toBeNull(); + expect(trustedServer).not.toHaveBeenCalled(); + + runner.dispatchEvent(new Event('load')); + await expect(accepted).resolves.toBe(true); + expect(slot.querySelector('span')).toBeNull(); + expect(frame.style.display).toBe(''); }); - it('contract test: declines missing, throwing, rejecting, and malformed hooks without fallback', async () => { + it('fails closed when the runner fails without clearing publisher content', async () => { const trustedServer = vi.fn(() => true); - await expect( - dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) - ).resolves.toBe(false); - - window.tsjs = { - apsNativeRenderer: { - render: () => { - throw new Error('fictional'); - }, - }, - } as typeof window.tsjs; - await expect( - dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) - ).resolves.toBe(false); - - window.tsjs = { - apsNativeRenderer: { render: () => Promise.reject(new Error('fictional')) }, - } as typeof window.tsjs; - await expect( - dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) - ).resolves.toBe(false); - - window.tsjs = { - apsNativeRenderer: { render: () => ({ accepted: false }) }, - } as typeof window.tsjs; - await expect( - dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) - ).resolves.toBe(false); + const accepted = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer, + }); + const frame = document.querySelector('#fictional-slot iframe')!; + const { runner } = nativeRunnerState(frame); - window.tsjs = { - apsNativeRenderer: { render: () => ({ accepted: 'yes' }) }, - } as typeof window.tsjs; - await expect( - dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) - ).resolves.toBe(false); + runner.dispatchEvent(new Event('error')); + await expect(accepted).resolves.toBe(false); expect(trustedServer).not.toHaveBeenCalled(); - expect(document.querySelector('iframe')).toBeNull(); + expect(document.querySelector('#fictional-slot iframe')).toBeNull(); + expect(document.querySelector('#fictional-slot span')).not.toBeNull(); }); - it('contract test: ignores a stale acknowledgement after a replacement dispatch', async () => { - let resolveFirst: ((value: { accepted: boolean }) => void) | undefined; - const render = vi - .fn() - .mockImplementationOnce( - () => - new Promise((resolve) => { - resolveFirst = resolve; - }) - ) - .mockResolvedValueOnce({ accepted: true }); - window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; - + it('cancels a pending runner when a newer dispatch replaces it', async () => { const first = dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer: () => true, }); + const firstFrame = document.querySelector('#fictional-slot iframe')!; + const second = dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer: () => true, }); - resolveFirst!({ accepted: true }); + const secondFrame = document.querySelector('#fictional-slot iframe')!; + expect(firstFrame.isConnected).toBe(false); + expect(secondFrame).not.toBe(firstFrame); await expect(first).resolves.toBe(false); + nativeRunnerState(secondFrame).runner.dispatchEvent(new Event('load')); await expect(second).resolves.toBe(true); }); - it('contract test: a missing hook supersedes an older pending dispatch', async () => { - let resolveFirst: ((value: { accepted: boolean }) => void) | undefined; - const render = vi.fn( - () => - new Promise<{ accepted: boolean }>((resolve) => { - resolveFirst = resolve; - }) - ); - window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; - + it('lets an invalid replacement cancel an older pending runner', async () => { const first = dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer: () => true, }); - delete window.tsjs!.apsNativeRenderer; const second = dispatchApsRendering({ slotId: 'fictional-slot', - renderer: descriptor(), + renderer: descriptor({ aaxResponse: 'invalid' }), trustedServer: () => true, }); - resolveFirst!({ accepted: true }); - await expect(second).resolves.toBe(false); + expect(second).toBe(false); await expect(first).resolves.toBe(false); + expect(document.querySelector('#fictional-slot iframe')).toBeNull(); + expect(document.querySelector('#fictional-slot span')).not.toBeNull(); }); - it('contract test: a trusted-server dispatch supersedes an older native dispatch', async () => { - let resolveFirst: ((value: { accepted: boolean }) => void) | undefined; - window.tsjs = { - apsNativeRenderer: { - render: () => - new Promise<{ accepted: boolean }>((resolve) => { - resolveFirst = resolve; - }), - }, - } as typeof window.tsjs; + it('lets a trusted-server dispatch supersede an older native frame', async () => { const first = dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), @@ -440,87 +410,64 @@ describe('publisher-native APS hook contract tests', () => { renderer: descriptor(), trustedServer, }); - resolveFirst!({ accepted: true }); expect(second).toBe(true); expect(trustedServer).toHaveBeenCalledOnce(); await expect(first).resolves.toBe(false); + expect(document.querySelector('#fictional-slot iframe')).toBeNull(); }); - it('contract test: contains throwing hook and acknowledgement accessors', async () => { + it('resolves a logical GPT slot through the injected div mapping', async () => { + document.body.innerHTML = '
existing
'; + window.tsjs = { divToSlotId: { 'div-header': 'homepage_header' } } as typeof window.tsjs; + + const accepted = dispatchApsRendering({ + slotId: 'homepage_header', + renderer: descriptor(), + trustedServer: () => true, + }); + const frame = document.querySelector('#div-header iframe')!; + nativeRunnerState(frame).runner.dispatchEvent(new Event('load')); + + await expect(accepted).resolves.toBe(true); + expect(document.querySelector('#div-header span')).toBeNull(); + }); + + it('contains throwing publisher slot mappings without falling back', async () => { const tsjs = {} as NonNullable; - Object.defineProperty(tsjs, 'apsNativeRenderer', { + Object.defineProperty(tsjs, 'divToSlotId', { get: () => { - throw new Error('fictional hook lookup failure'); + throw new Error('fictional mapping lookup failure'); }, }); window.tsjs = tsjs; + const trustedServer = vi.fn(() => true); await expect( dispatchApsRendering({ - slotId: 'fictional-slot', - renderer: descriptor(), - trustedServer: () => true, - }) - ).resolves.toBe(false); - - window.tsjs = { - apsNativeRenderer: { - render: () => - new Proxy( - { accepted: true }, - { - ownKeys: () => { - throw new Error('fictional acknowledgement inspection failure'); - }, - } - ), - }, - } as typeof window.tsjs; - await expect( - dispatchApsRendering({ - slotId: 'fictional-slot', + slotId: 'logical-slot', renderer: descriptor(), - trustedServer: () => true, + trustedServer, }) ).resolves.toBe(false); + expect(trustedServer).not.toHaveBeenCalled(); + expect(document.querySelector('iframe')).toBeNull(); }); - it('contract test: times out a hook and ignores its late acknowledgement', async () => { + it('times out an unacknowledged runner without clearing publisher content', async () => { vi.useFakeTimers(); try { - let resolveHook: ((value: { accepted: boolean }) => void) | undefined; - window.tsjs = { - apsNativeRenderer: { - render: () => - new Promise<{ accepted: boolean }>((resolve) => { - resolveHook = resolve; - }), - }, - } as typeof window.tsjs; - const result = dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer: () => true, }); - await vi.advanceTimersByTimeAsync(APS_NATIVE_RENDERER_ACK_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(APS_NATIVE_RENDERER_TIMEOUT_MS); await expect(result).resolves.toBe(false); expect(vi.getTimerCount()).toBe(0); - resolveHook!({ accepted: true }); - await Promise.resolve(); - - window.tsjs = { - apsNativeRenderer: { render: () => ({ accepted: true }) }, - } as typeof window.tsjs; - await expect( - dispatchApsRendering({ - slotId: 'fictional-slot', - renderer: descriptor(), - trustedServer: () => true, - }) - ).resolves.toBe(true); + expect(document.querySelector('#fictional-slot iframe')).toBeNull(); + expect(document.querySelector('#fictional-slot span')).not.toBeNull(); } finally { vi.useRealTimers(); } diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 7f23cd394..b8f1bfd93 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -7,7 +7,10 @@ import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vites import envelope from '../../fixtures/aps-renderer-v1.json'; import type { AuctionBidData, TsjsApi } from '../../../src/core/types'; -import { APS_RENDERING_MODE_META_NAME } from '../../../src/integrations/aps/render'; +import { + APS_PREBID_CREATIVE_RUNNER_URL, + APS_RENDERING_MODE_META_NAME, +} from '../../../src/integrations/aps/render'; function enablePublisherNativeMode(): HTMLMetaElement { const marker = document.createElement('meta'); @@ -17,6 +20,26 @@ function enablePublisherNativeMode(): HTMLMetaElement { return marker; } +function nativeRunnerIn(divId: string): { + frame: HTMLIFrameElement; + runner: HTMLScriptElement; + event: CustomEvent<{ aaxResponse: string; seatBidId: string }>; +} { + const container = document.getElementById(divId)!; + const frame = Array.from(container.querySelectorAll('iframe')).find( + (candidate) => candidate.title === 'Ad content' + ); + expect(frame).not.toBeUndefined(); + const runner = frame!.contentDocument?.querySelector('script'); + const frameWindow = frame!.contentWindow as unknown as { + _aps: Map> }>; + }; + const event = Array.from(frameWindow._aps.values())[0]?.queue[0]; + expect(runner?.src).toBe(APS_PREBID_CREATIVE_RUNNER_URL); + expect(event).not.toBeUndefined(); + return { frame: frame!, runner: runner!, event }; +} + function apsRenderer() { const bid = envelope.seatbid[0].bid[0]; return { @@ -3206,14 +3229,12 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); - it('contract test: delegates a server APS owner to the native hook without a Universal Creative response', async () => { + it('contract test: renders a server APS owner with the injected runner and no Universal Creative response', async () => { const renderer = apsRenderer(); - const render = vi.fn().mockResolvedValue({ accepted: true }); (window as TestWindow).tsjs.bids.homepage_header = { hb_adid: renderer.bidId, renderer, }; - (window as TestWindow).tsjs.apsNativeRenderer = { render }; const marker = enablePublisherNativeMode(); try { @@ -3229,15 +3250,17 @@ describe('installTsRenderBridge', () => { bridgeListener(request); bridgeListener(request); + const native = nativeRunnerIn('div-header'); + expect(native.event.type).toBe('prebid/creative/render'); + expect(native.event.detail).toEqual({ + aaxResponse: renderer.aaxResponse, + seatBidId: renderer.bidId, + }); + native.runner.dispatchEvent(new Event('load')); await Promise.resolve(); await Promise.resolve(); - expect(render).toHaveBeenCalledOnce(); - expect(render).toHaveBeenCalledWith({ - version: 1, - slotId: 'homepage_header', - renderer, - }); + expect(native.frame.style.display).toBe(''); expect(portMessages).toEqual([]); expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); } finally { @@ -3245,14 +3268,12 @@ describe('installTsRenderBridge', () => { } }); - it('contract test: declines a server APS owner without a Universal Creative response or fallback', async () => { + it('contract test: fails a server APS runner without a Universal Creative response or fallback', async () => { const renderer = apsRenderer(); - const render = vi.fn().mockResolvedValue({ accepted: false }); (window as TestWindow).tsjs.bids.homepage_header = { hb_adid: renderer.bidId, renderer, }; - (window as TestWindow).tsjs.apsNativeRenderer = { render }; const marker = enablePublisherNativeMode(); try { @@ -3267,12 +3288,13 @@ describe('installTsRenderBridge', () => { }) as unknown as MessageEvent; bridgeListener(request); + nativeRunnerIn('div-header').runner.dispatchEvent(new Event('error')); await Promise.resolve(); await Promise.resolve(); bridgeListener(request); - expect(render).toHaveBeenCalledOnce(); expect(portMessages).toEqual([]); + expect(document.querySelector('iframe[title="Ad content"]')).toBeNull(); expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); } finally { marker.remove(); @@ -3334,11 +3356,10 @@ describe('installTsRenderBridge', () => { foreignIframe.remove(); }); - it('contract test: declines a registered APS capability without a Universal Creative response or markUsed', async () => { + it('contract test: fails a registered APS runner without a Universal Creative response or markUsed', async () => { const renderer = apsRenderer(); const prebidAdId = 'native-prebid-decline-ad-id'; const markUsed = vi.fn(); - const render = vi.fn().mockResolvedValue({ accepted: false }); (window as TestWindow).tsjs.apsPrebidRenderers = { [prebidAdId]: { adUnitCode: 'div-header', @@ -3348,7 +3369,6 @@ describe('installTsRenderBridge', () => { markUsed, }, }; - (window as TestWindow).tsjs.apsNativeRenderer = { render }; const marker = enablePublisherNativeMode(); try { @@ -3363,25 +3383,25 @@ describe('installTsRenderBridge', () => { }) as unknown as MessageEvent; bridgeListener(request); + nativeRunnerIn('div-header').runner.dispatchEvent(new Event('error')); await Promise.resolve(); await Promise.resolve(); bridgeListener(request); - expect(render).toHaveBeenCalledOnce(); expect(markUsed).not.toHaveBeenCalled(); expect(portMessages).toEqual([]); expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); + expect(document.querySelector('iframe[title="Ad content"]')).toBeNull(); expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); } finally { marker.remove(); } }); - it('contract test: consumes a registered APS capability and marks it used only after native acceptance', async () => { + it('contract test: consumes a registered APS capability and marks it used only after runner load', async () => { const renderer = apsRenderer(); const prebidAdId = 'native-prebid-ad-id'; const markUsed = vi.fn(); - const render = vi.fn().mockResolvedValue({ accepted: true }); (window as TestWindow).tsjs.apsPrebidRenderers = { [prebidAdId]: { adUnitCode: 'div-header', @@ -3391,7 +3411,6 @@ describe('installTsRenderBridge', () => { markUsed, }, }; - (window as TestWindow).tsjs.apsNativeRenderer = { render }; const marker = enablePublisherNativeMode(); try { @@ -3407,11 +3426,13 @@ describe('installTsRenderBridge', () => { bridgeListener(request); expect(markUsed).not.toHaveBeenCalled(); + const native = nativeRunnerIn('div-header'); + native.runner.dispatchEvent(new Event('load')); await Promise.resolve(); await Promise.resolve(); bridgeListener(request); - expect(render).toHaveBeenCalledOnce(); + expect(native.frame.style.display).toBe(''); expect(markUsed).toHaveBeenCalledOnce(); expect(portMessages).toEqual([]); expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index 9a5f4cb7e..1479e17f8 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -36,7 +36,7 @@ debug = false # inventory_domain = "publisher.example" # inventory_page_origin = "https://www.publisher.example" allow_script_creatives = false -# Default. Set publisher_native only with a publisher-installed hook (below). +# Default. Set publisher_native only for the controlled friendly-frame experiment below. rendering_mode = "trusted_server" [auction] @@ -51,30 +51,27 @@ timeout_ms = 2000 `allow_script_creatives` defaults to `false`. While disabled, APS script bids are rejected before per-impression reduction, floors, mediation, and winner selection. Enable it only for a controlled cohort after the browser-security checks in [Rollout](#rollout) pass. -`rendering_mode` is a strict enum: `trusted_server` (the default) retains the opaque static renderer route, and `publisher_native` disables that route and emits the inert `` head marker selecting the publisher hook below. The marker works under a publisher CSP that blocks inline scripts. Unknown values fail configuration deserialization. +`rendering_mode` is a strict enum: `trusted_server` (the default) retains the opaque static renderer route, and `publisher_native` disables that route and emits the inert `` head marker selecting the injected friendly-frame runner below. The marker works under a publisher CSP that blocks inline scripts. Unknown values fail configuration deserialization. -### Publisher-native hook experiment +### Publisher-native runner experiment -`publisher_native` is an opt-in publisher integration seam, **not** APS compatibility proof. No public APS API was found that accepts an externally selected OpenRTB `aaxResponse` for native rendering. In particular, `apstag.setDisplayBids()` operates on APS's own `fetchBids()` state and is not an ingestion API for the exact Trusted Server-selected bid. Trusted Server does not call `apstag`, `fetchBids`, or `setDisplayBids`, mutate APS internals, or start a second auction. +`publisher_native` is an opt-in browser experiment, **not** general APS compatibility proof. No public `apstag` API was found that accepts an externally selected OpenRTB `aaxResponse`. In controlled browser testing, `apstag.renderImp(document, bidId)` did not render the Trusted Server bid because that bid was absent from the SDK's browser-auction state. Trusted Server therefore does not call `apstag`, `fetchBids`, or `setDisplayBids`, mutate the publisher's APS SDK, or start a second auction. Instead, this mode reuses the same `prebid/creative/render` runner contract already used by `trusted_server` mode, but inside a publisher-origin frame; that observed vendor contract still requires APS account-team validation. -Before Trusted Server JS receives a selected descriptor, the publisher must install this versioned hook: +No publisher JavaScript change is required. After validating and freezing the exact selected descriptor, Trusted Server JS: -```js -window.tsjs = window.tsjs || {} -window.tsjs.apsNativeRenderer = { - render({ version, slotId, renderer }) { - // version is exactly 1; renderer is frozen and fully validated. - // Render only this exact selected descriptor through publisher-owned logic. - return { accepted: true } - }, -} -``` +1. resolves the direct-auction slot or its injected GAM div mapping; +2. creates a hidden, publisher-origin friendly iframe sized to the winner; +3. initializes only that fresh frame's account-scoped `_aps` event queue; +4. queues `prebid/creative/render` with the selected `aaxResponse` and bid ID; and +5. loads the fixed `https://client.aps.amazon-adsystem.com/prebid-creative.js` runner. + +The existing publisher content remains visible until the runner script loads. A runner error, a blocked script, a missing slot, a superseding dispatch, or a load taking longer than 10 seconds removes the pending frame and visibly declines the bid. It never falls back to `/integrations/aps/renderer` or sends a Universal Creative renderer response. Trusted Server treats runner load as successful handoff; the runner owns subsequent creative completion and resource loading. -The hook receives exactly `{ version: 1, slotId, renderer }`. It must return or resolve an object with `accepted: boolean`; `reason?: string` is allowed for fictional-safe diagnostics. Missing hooks, throws, rejected promises, malformed acknowledgements, `{ accepted: false }`, and acknowledgements that take longer than 10 seconds visibly decline the bid. They never fall back to the Trusted Server iframe or send a Universal Creative renderer response. A newer dispatch makes an older acknowledgement stale and ignored; late acknowledgements after the timeout are also ignored. Hook implementations own cancellation of any already-started side effect and actual render completion. +Unlike `trusted_server` mode, this friendly frame deliberately has no opaque-origin sandbox. The fixed APS runner and its creative execute with the behavior of a publisher-origin integration, so `publisher_native` has a larger security surface—especially when `allow_script_creatives = true`. Use only a controlled cohort, and ensure publisher CSP permits the APS runner and required creative resources. -For a client-side Prebid APS capability, Trusted Server consumes the one-shot capability before delegation and calls `markWinningBidAsUsed` only after `accepted: true`. For server/GPT ownership, it similarly claims the slot/ad ID before invoking the hook. This prevents native and Trusted Server rendering from both owning the same response, but an accepting hook is responsible for real rendering semantics. +For a client-side Prebid APS capability, Trusted Server consumes the one-shot capability before starting the runner and calls `markWinningBidAsUsed` only after the runner loads. For server/GPT ownership, it similarly claims the slot/ad ID first. This prevents native and Trusted Server rendering from both owning the same response. -Disable or coordinate publisher-native APS demand for every `publisher_native` cohort. Otherwise native APS demand and this server-selected bid can duplicate demand. Validate a controlled real publisher/account setup with the APS account team before any production rollout. +Disable or coordinate existing publisher-native APS demand for every `publisher_native` cohort. Otherwise the publisher's normal APS auction and this server-selected bid can duplicate demand. Validate the exact account, inventory, CSP, iframe/script creative behavior, impression reporting, and click-through behavior with the APS account team before any production rollout. Set `inventory_domain` and `inventory_page_origin` together only when the public deployment hostname differs from the inventory identity authorized by APS. The domain becomes `site.domain`. The HTTPS page origin replaces the current page's scheme and host while preserving its path; query and fragment data are removed before forwarding. The origin must be the inventory domain or one of its subdomains and cannot include credentials, a port, path, query, or fragment. These values come only from operator configuration; Trusted Server never accepts APS inventory identity from the client auction payload. @@ -194,11 +191,11 @@ It deliberately omits `allow-same-origin`, so APS and bidder execution remains b ### Direct `/auction` -In `trusted_server` mode, the TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. In `publisher_native` mode it instead calls the explicit publisher hook and creates no Trusted Server iframe. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. +In `trusted_server` mode, the TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. In `publisher_native` mode it creates the injected friendly iframe and queues the response for the fixed APS Prebid creative runner. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. ### GAM and Universal Creative -For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. In `publisher_native` mode the ownership-checked bridge calls the publisher hook without sending a Universal Creative renderer response; in `trusted_server` mode it uses the static dynamic renderer described below. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program that creates the same opaque renderer iframe. +For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. In `publisher_native` mode the ownership-checked bridge resolves the publisher div and starts the friendly-frame runner without sending a Universal Creative renderer response; in `trusted_server` mode it uses the static dynamic renderer described below. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program that creates the same opaque renderer iframe. For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes the capability once, and passes the APS bid ID separately to the Amazon runner. @@ -242,11 +239,12 @@ Use fictional values in source-controlled configuration and fixtures. Supply con 1. Obtain APS account-team confirmation for edge-originated OpenRTB traffic. 2. Enable Trusted Server APS only for an isolated cohort and disable native APS demand there. -3. Keep `allow_script_creatives = false` and observe iframe bids through direct and GAM paths. +3. Keep the default `trusted_server` mode and `allow_script_creatives = false`; observe iframe bids through direct and GAM paths. 4. Confirm outbound privacy fields, aggregate diagnostics, decoded-price competition, line-item targeting, dimensions, click-throughs, and opaque-origin isolation. -5. Run the restrictive-CSP browser proof for script behavior. -6. Only then enable script creatives for the isolated cohort and validate them in a real browser. -7. Expand traffic only after APS confirmation and successful controlled validation. +5. In a still-smaller cohort, set `rendering_mode = "publisher_native"` and confirm the fixed runner request, friendly-frame dimensions, iframe creatives, impression reporting, and click-throughs without a request to `/integrations/aps/renderer`. +6. Confirm the publisher CSP permits the runner but does not need to permit inline Trusted Server scripts. +7. Only after reviewing the friendly-frame security tradeoff, enable script creatives for the isolated native cohort and validate them in a real browser. +8. Expand traffic only after APS confirmation and successful controlled validation. ## Troubleshooting @@ -262,12 +260,12 @@ Use fictional values in source-controlled configuration and fixtures. Supply con ### Winner targets but does not render -- Confirm `GET /integrations/aps/renderer` returns HTML with its CSP and `Referrer-Policy: no-referrer`. -- Confirm publisher CSP permits `frame-src 'self'`. +- In `trusted_server` mode, confirm `GET /integrations/aps/renderer` returns HTML with its CSP and `Referrer-Policy: no-referrer`, and that publisher CSP permits `frame-src 'self'`. +- In `publisher_native` mode, confirm the mode `` marker is present, the slot receives a hidden friendly iframe, and `https://client.aps.amazon-adsystem.com/prebid-creative.js` is not blocked by CSP. The static renderer route is intentionally absent in this mode. - Confirm the GAM creative uses the supported Prebid Universal Creative bridge and the winning `hb_adid`. - For client-side `trustedServer` adapter auctions, confirm Prebid's `bidResponse` contains a generated `adId` and that the corresponding capability appears briefly in `window.tsjs.apsPrebidRenderers` before rendering. -- Ensure no native APS path is trying to handle the same cohort. -- Keep script creatives disabled while diagnosing iframe rendering. +- Ensure no publisher APS auction is trying to handle the same cohort. +- Keep script creatives disabled while diagnosing either rendering mode. ## Verification diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 6e844e385..cedeb5f6e 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -156,7 +156,7 @@ debug = false # Script creatives require separate security validation before opt-in. allow_script_creatives = false # Default: Trusted Server's opaque static renderer route. Set publisher_native only -# when the publisher installs the documented tsjs.apsNativeRenderer hook. +# for the controlled publisher-origin friendly-frame experiment. rendering_mode = "trusted_server" [integrations.google_tag_manager] From caf44220823d6a90061ee969bf14e397d04a75d9 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 22:38:56 +0530 Subject: [PATCH 375/395] Reserve the full admin namespace at the fallback boundary A percent-encoded separator such as `/_ts/admin%2Fec` matches the documented `^/_ts/admin` basic-auth handler, so the request is authenticated, but the literal-slash namespace check missed it. The request then reached publisher fallback, which forwards the caller's `Authorization` header and body to the publisher origin. Reserve the whole `/_ts/admin` prefix instead, mirroring the auth handler regex, and evaluate it on the percent-decoded path as well so `%2F`, `%2f`, and double-encoded forms stay closed. Reserve the retired `/admin/keys` alias namespace the same way: only the two exact paths are routed to a local deny, so trailing, descendant, and encoded-separator forms previously fell through with any supplied credentials and body. Every adapter shares this boundary, so the fix and its cross-adapter regressions land in one place. --- .../tests/routes.rs | 12 ++ .../tests/routes.rs | 12 ++ .../trusted-server-adapter-fastly/src/app.rs | 12 ++ .../tests/routes.rs | 12 ++ crates/trusted-server-core/src/auth.rs | 23 +++ crates/trusted-server-core/src/ec/admin.rs | 140 ++++++++++++++++-- docs/guide/api-reference.md | 2 +- docs/guide/architecture.md | 14 +- 8 files changed, 205 insertions(+), 22 deletions(-) diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index a0452e255..d9f0f6b5a 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -413,6 +413,18 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { "/_ts/admin/eids.json".to_owned(), "/_ts/admin/ec;foo".to_owned(), format!("/_ts/admin/ec%2F{ec_id}"), + // Percent-encoded separators match the `^/_ts/admin` basic-auth + // handler but not a literal-slash namespace check, so they must be + // reserved before publisher fallback forwards credentials upstream. + "/_ts/admin%2Fec".to_owned(), + "/_ts/admin%2fec".to_owned(), + // Retired non-`/_ts` alias namespace: only the two exact paths are + // routed to a local deny, so descendants and encoded separators must + // be reserved at the shared fallback boundary. + "/admin/keys".to_owned(), + "/admin/keys/rotate/extra".to_owned(), + "/admin/keys%2Frotate".to_owned(), + "/admin%2fkeys/rotate".to_owned(), ] { for method in ["GET", "POST"] { let request = Request::builder() diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index 528d9348e..1ad07bcdc 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -396,6 +396,18 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { "/_ts/admin/eids.json".to_owned(), "/_ts/admin/ec;foo".to_owned(), format!("/_ts/admin/ec%2F{ec_id}"), + // Percent-encoded separators match the `^/_ts/admin` basic-auth + // handler but not a literal-slash namespace check, so they must be + // reserved before publisher fallback forwards credentials upstream. + "/_ts/admin%2Fec".to_owned(), + "/_ts/admin%2fec".to_owned(), + // Retired non-`/_ts` alias namespace: only the two exact paths are + // routed to a local deny, so descendants and encoded separators must + // be reserved at the shared fallback boundary. + "/admin/keys".to_owned(), + "/admin/keys/rotate/extra".to_owned(), + "/admin/keys%2Frotate".to_owned(), + "/admin%2fkeys/rotate".to_owned(), ] { for method in ["GET", "POST"] { let request = request_builder() diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 072208fb2..000768666 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1894,6 +1894,18 @@ mod tests { "/_ts/admin/eids.json".to_owned(), "/_ts/admin/ec;foo".to_owned(), format!("/_ts/admin/ec%2F{ec_id}"), + // Percent-encoded separators match the `^/_ts/admin` basic-auth + // handler but not a literal-slash namespace check, so they must be + // reserved before publisher fallback forwards credentials upstream. + "/_ts/admin%2Fec".to_owned(), + "/_ts/admin%2fec".to_owned(), + // Retired non-`/_ts` alias namespace: only the two exact paths are + // routed to a local deny, so descendants and encoded separators must + // be reserved at the shared fallback boundary. + "/admin/keys".to_owned(), + "/admin/keys/rotate/extra".to_owned(), + "/admin/keys%2Frotate".to_owned(), + "/admin%2fkeys/rotate".to_owned(), ] { for method in [Method::GET, Method::POST] { let request = request_builder() diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index e6737b6ba..b130b9a28 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -231,6 +231,18 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { "/_ts/admin/eids.json".to_owned(), "/_ts/admin/ec;foo".to_owned(), format!("/_ts/admin/ec%2F{ec_id}"), + // Percent-encoded separators match the `^/_ts/admin` basic-auth + // handler but not a literal-slash namespace check, so they must be + // reserved before publisher fallback forwards credentials upstream. + "/_ts/admin%2Fec".to_owned(), + "/_ts/admin%2fec".to_owned(), + // Retired non-`/_ts` alias namespace: only the two exact paths are + // routed to a local deny, so descendants and encoded separators must + // be reserved at the shared fallback boundary. + "/admin/keys".to_owned(), + "/admin/keys/rotate/extra".to_owned(), + "/admin/keys%2Frotate".to_owned(), + "/admin%2fkeys/rotate".to_owned(), ] { for method in ["GET", "POST"] { let request = request_builder() diff --git a/crates/trusted-server-core/src/auth.rs b/crates/trusted-server-core/src/auth.rs index ecc2fdb8f..6c92d042d 100644 --- a/crates/trusted-server-core/src/auth.rs +++ b/crates/trusted-server-core/src/auth.rs @@ -125,6 +125,29 @@ mod tests { ); } + #[test] + fn encoded_admin_separator_path_is_auth_gated() { + // `^/_ts/admin` matches the raw path, so a percent-encoded separator + // still consumes admin credentials. The publisher-fallback boundary + // reserves the same paths so those credentials are never forwarded + // upstream (see `ec::admin::deny_admin_diagnostic_fallback`). + let settings = create_test_settings(); + + for path in ["/_ts/admin%2Fec", "/_ts/admin%2fec"] { + let req = build_request(Method::GET, &format!("https://example.com{path}")); + + let response = enforce_basic_auth(&settings, &req) + .expect("should evaluate auth") + .unwrap_or_else(|| panic!("should challenge {path}")); + + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "should require credentials for {path}" + ); + } + } + #[test] fn no_challenge_for_non_protected_path() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 96fb708b6..a94679781 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -16,6 +16,7 @@ //! auth-gated and operator-facing, responses intentionally include full //! internal detail (raw consent strings, partner UIDs, parse errors). +use std::borrow::Cow; use std::collections::BTreeMap; use http::{HeaderValue, Method, Request, Response, StatusCode, header}; @@ -47,6 +48,22 @@ const ADMIN_EC_PATH: &str = "/_ts/admin/ec"; /// Route used by the request-only EID cookie diagnostic. const ADMIN_EIDS_PATH: &str = "/_ts/admin/eids"; +/// Reserved Trusted Server admin prefix. +/// +/// Mirrors the documented `^/_ts/admin` basic-auth handler regex, so every +/// path that handler authenticates is also reserved at the fallback boundary. +/// Matching on the bare prefix — rather than on `/_ts/admin` plus a literal +/// `/` — also covers percent-encoded separators such as `/_ts/admin%2Fec`, +/// which the auth handler matches but a literal-slash check does not. +const ADMIN_NAMESPACE_PREFIX: &str = "/_ts/admin"; + +/// Retired non-`/_ts` admin key alias prefix. +/// +/// The exact `/admin/keys/rotate` and `/admin/keys/deactivate` aliases are +/// routed to a local deny by each adapter; the rest of the retired namespace +/// (trailing, descendant, and encoded-separator forms) is reserved here. +const RETIRED_ADMIN_KEYS_PREFIX: &str = "/admin/keys"; + #[derive(Debug, Clone, Copy, Eq, PartialEq)] enum AdminDiagnosticShape { ValidResource, @@ -66,15 +83,36 @@ fn admin_diagnostic_shape(path: &str) -> Option { }); } - if path.starts_with("/_ts/admin/eids/") { - return Some(AdminDiagnosticShape::Malformed); - } - // Reserve the complete admin namespace at the publisher-fallback boundary. // A successfully authenticated malformed or future admin path must never // forward its Authorization header or body to the publisher origin. - (path == "/_ts/admin" || path.starts_with("/_ts/admin/")) - .then_some(AdminDiagnosticShape::Malformed) + let reserved = is_reserved_admin_path(path) + || percent_decoded_path(path).is_some_and(|decoded| is_reserved_admin_path(&decoded)); + + reserved.then_some(AdminDiagnosticShape::Malformed) +} + +/// Returns whether `path` sits in a namespace that must never reach publisher +/// fallback, because doing so would forward Trusted Server admin credentials +/// and request bodies to the publisher origin. +fn is_reserved_admin_path(path: &str) -> bool { + path.starts_with(ADMIN_NAMESPACE_PREFIX) || path.starts_with(RETIRED_ADMIN_KEYS_PREFIX) +} + +/// Percent-decodes `path` once, returning `None` when the path contains no +/// escape sequence or decodes to invalid UTF-8. +/// +/// Routers and the basic-auth matcher both operate on the raw path, so an +/// encoded separator can shift a request out of the literal admin namespace +/// while still matching the admin auth handler. Checking the decoded form as +/// well keeps the reservation closed for `%2F`, `%2f`, and their +/// double-encoded variants. +fn percent_decoded_path(path: &str) -> Option { + if !path.contains('%') { + return None; + } + + urlencoding::decode(path).ok().map(Cow::into_owned) } /// Returns a local denial response when an admin diagnostic request reaches @@ -82,8 +120,11 @@ fn admin_diagnostic_shape(path: &str) -> Option { /// /// Valid diagnostic resources reject non-GET methods with `405 Method Not /// Allowed`. Malformed, trailing, unknown, and any valid GET admin route that -/// unexpectedly reaches fallback return `404 Not Found`. Paths outside the -/// reserved `/_ts/admin` namespace return `None`, preserving normal fallback. +/// unexpectedly reaches fallback return `404 Not Found`. The reservation +/// spans the whole `/_ts/admin` prefix — including percent-encoded separators +/// such as `/_ts/admin%2Fec` — plus the retired `/admin/keys` alias namespace, +/// evaluated on both the raw and the percent-decoded path. Paths outside those +/// namespaces return `None`, preserving normal fallback. #[must_use] pub fn deny_admin_diagnostic_fallback(req: &Request) -> Option> { let shape = admin_diagnostic_shape(req.uri().path())?; @@ -748,13 +789,84 @@ mod tests { } #[test] - fn admin_diagnostic_fallback_ignores_unrelated_publisher_paths() { - let request = request_with_method(http::Method::POST, "/articles/example"); + fn admin_diagnostic_fallback_reserves_encoded_admin_separators() { + // `/_ts/admin%2Fec` matches the documented `^/_ts/admin` basic-auth + // handler, so it is authenticated, but a literal-slash namespace check + // misses it. Reaching publisher fallback would forward the caller's + // `Authorization` header and body to the origin. + let paths = [ + "/_ts/admin%2Fec", + "/_ts/admin%2fec", + "/_ts/admin%2Fkeys/rotate", + "/_ts/admin%252Fec", + "/_ts/admin%5Cec", + "/_ts/adminec", + "/%5Fts/admin/ec", + ]; - assert!( - deny_admin_diagnostic_fallback(&request).is_none(), - "should leave unrelated publisher fallback unchanged" - ); + for path in paths { + for method in [http::Method::GET, http::Method::POST] { + let request = request_with_method(method.clone(), path); + let response = deny_admin_diagnostic_fallback(&request) + .unwrap_or_else(|| panic!("should deny {method} {path} locally")); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should deny {path} before publisher fallback" + ); + } + } + } + + #[test] + fn admin_diagnostic_fallback_reserves_retired_admin_keys_namespace() { + // The retired non-`/_ts` aliases are not covered by the `^/_ts/admin` + // basic-auth handler. Only the two exact paths are routed to a local + // deny, so trailing, descendant, and encoded-separator forms must be + // denied at the shared fallback boundary instead. + let paths = [ + "/admin/keys", + "/admin/keys/", + "/admin/keys/rotate/", + "/admin/keys/rotate/extra", + "/admin/keys%2Frotate", + "/admin/keys%2frotate", + "/admin%2Fkeys/rotate", + "/admin%2fkeys%2Frotate", + ]; + + for path in paths { + for method in [http::Method::GET, http::Method::POST] { + let request = request_with_method(method.clone(), path); + let response = deny_admin_diagnostic_fallback(&request) + .unwrap_or_else(|| panic!("should deny {method} {path} locally")); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should deny {path} before publisher fallback" + ); + } + } + } + + #[test] + fn admin_diagnostic_fallback_ignores_unrelated_publisher_paths() { + for path in [ + "/articles/example", + "/admin", + "/admin/login", + "/admin/keyboards", + "/_ts/api/v1/batch-sync", + ] { + let request = request_with_method(http::Method::POST, path); + + assert!( + deny_admin_diagnostic_fallback(&request).is_none(), + "should leave unrelated publisher fallback unchanged for {path}" + ); + } } #[test] diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 0fff97f8f..b4cb64481 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -583,7 +583,7 @@ curl -X POST https://edge.example.com/_ts/admin/keys/deactivate \ ## Admin Diagnostic Endpoints -These endpoints expose sensitive identity and cookie data and require HTTP Basic Authentication. Configure a handler that covers the entire `/_ts/admin` namespace; startup rejects configurations that do not protect every admin route. Normal diagnostic-handler responses after successful authentication are JSON with `Cache-Control: no-store`. Missing or invalid credentials receive the shared plaintext `401 Unauthorized` Basic-auth challenge. Unexpected configuration or KV failures use the adapter's shared plaintext `5xx` error response. Those authentication and internal-error responses are outside the diagnostic JSON and cache-header contract. +These endpoints expose sensitive identity and cookie data and require HTTP Basic Authentication. Configure a handler that covers the entire `/_ts/admin` namespace; startup rejects configurations that do not protect every admin route, including handlers that match only some `/_ts/admin/ec/{id}` values — the dynamic route needs a prefix-level matcher such as `^/_ts/admin` or `^/_ts/admin/ec/`. The whole `/_ts/admin` prefix is reserved: any admin path that reaches publisher fallback — unknown, malformed, or percent-encoded (`/_ts/admin%2Fec`) — is answered locally with `404` and is never proxied, so an admin `Authorization` header and request body never reach the publisher origin. The retired non-`/_ts` `/admin/keys` aliases are reserved the same way. Normal diagnostic-handler responses after successful authentication are JSON with `Cache-Control: no-store`. Missing or invalid credentials receive the shared plaintext `401 Unauthorized` Basic-auth challenge. Unexpected configuration or KV failures use the adapter's shared plaintext `5xx` error response. Those authentication and internal-error responses are outside the diagnostic JSON and cache-header contract. The examples below use fictional IDs and values only. diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 3b20000ec..da1a58bcd 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -53,13 +53,13 @@ Native Axum dev/test adapter (native binary): **Current limitations compared to the Fastly adapter:** -| Feature | Axum dev server | -| ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | -| KV store | Unavailable — synthetic-ID and consent routes degrade gracefully | -| Geo lookup | Always returns `None` | -| Config/secret-store writes | Return an error (read-only via env vars) | -| Admin key management (`/_ts/admin/keys/*`) | Returns 501 Not Implemented. Legacy `/admin/keys/*` aliases are denied locally with 404 and are not proxied to the publisher fallback | -| Auction fan-out ordering | Requests run concurrently via `tokio::spawn`; `select` returns first-to-complete but does not replicate Fastly's priority-queue tie-breaking | +| Feature | Axum dev server | +| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| KV store | Unavailable — synthetic-ID and consent routes degrade gracefully | +| Geo lookup | Always returns `None` | +| Config/secret-store writes | Return an error (read-only via env vars) | +| Admin key management (`/_ts/admin/keys/*`) | Returns 501 Not Implemented. Retired `/admin/keys` aliases, including trailing, descendant, and percent-encoded forms, are denied locally with 404 and are not proxied to the publisher fallback | +| Auction fan-out ordering | Requests run concurrently via `tokio::spawn`; `select` returns first-to-complete but does not replicate Fastly's priority-queue tie-breaking | ### trusted-server-adapter-spin From 8684e1b0dcba4a7d4a70c01909bb9062a5d229db Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 22:39:10 +0530 Subject: [PATCH 376/395] Require prefix-level admin EC auth coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handler coverage for `/_ts/admin/ec/{id}` was checked against representative EC IDs. The router accepts any segment after `/_ts/admin/ec/` and basic auth runs on the raw path before routing, so a handler matching only some ID shapes passed startup while leaving the rest of the route surface — including malformed IDs, which still reach the admin handler — uncovered and fail-closed at runtime. Probe the bare prefix and a concrete ID together instead: the prefix rejects handlers anchored to an ID grammar, the concrete ID rejects handlers anchored to the prefix itself, and only prefix-level matchers satisfy both. Apply the placeholder and weak password check to every handler rather than to handlers inferred to cover an admin endpoint. Handler selection is first-match-wins, so a narrow handler can shadow the admin namespace for paths no probe enumerates. --- CHANGELOG.md | 3 +- crates/trusted-server-core/src/settings.rs | 167 +++++++++++++++++---- docs/guide/configuration.md | 17 +++ 3 files changed, 156 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4283dcbc8..aade766a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- **Breaking** — Admin Basic-auth coverage now includes `GET /_ts/admin/ec`, `GET /_ts/admin/ec/{id}`, and `GET /_ts/admin/eids`. Existing configurations whose `[[handlers]]` patterns protect only the key-management endpoints now fail startup; broaden coverage before deploying, preferably with a namespace-boundary pattern such as `^/_ts/admin(?:/|$)`. +- **Breaking** — Admin Basic-auth coverage now includes `GET /_ts/admin/ec`, `GET /_ts/admin/ec/{id}`, and `GET /_ts/admin/eids`. Existing configurations whose `[[handlers]]` patterns protect only the key-management endpoints now fail startup; broaden coverage before deploying, preferably with a namespace-boundary pattern such as `^/_ts/admin(?:/|$)`. Coverage of the dynamic `/_ts/admin/ec/{id}` route is no longer inferred from ID-shaped samples: the router accepts any segment after `/_ts/admin/ec/` and Basic Auth runs on the raw path before routing, so patterns anchored to the EC ID grammar (for example `^/_ts/admin/ec/[a-f0-9]{64}[.][A-Za-z0-9]{6}$`) are rejected in favor of a prefix-level matcher. Placeholder and well-known weak handler passwords (`changeme`, `password`, `admin`, `replace-with-…`) now fail startup on every handler rather than only on handlers inferred to cover an admin endpoint, because first-match-wins handler selection lets a narrow handler shadow the admin namespace. - **Breaking** — Replaced the legacy APS contextual integration with APS OpenRTB at `/e/pb/bid`. APS configuration now uses canonical `account_id` (`pub_id` remains a compatibility alias), no longer requires APS-specific slot IDs, and defaults script creative eligibility off. Operators must update the endpoint, disable native APS demand for Trusted Server cohorts, and prepare GAM/Universal Creative targeting for `hb_bidder=aps` before rollout. `aps` entries in Prebid bidder lists are logged and stripped. APS renderer winners now preserve the upstream bid `id`, omit `crid` when APS omits it, and carry `ext.trusted_server.renderer` instead of `adm`; external `/auction` consumers must support this response shape. - **Breaking** — All auction paths now forward only a validated publisher-owned page URL as `site.page`, removing query and fragment data. APS OpenRTB omits `site.ref`; the existing Prebid Server path continues to forward the browser `Referer` as `site.ref`. Query-driven sites may lose contextual targeting and per-page reporting signals that previously came from query parameters. - **Breaking** — `bid_param_zone_overrides` inner values must now be JSON objects; previously non-object or empty values (`"header" = "x"`, `"header" = {}`) were accepted and silently produced a dead rule at runtime. They now fail at startup with a configuration error. Operators upgrading should audit their `bid_param_zone_overrides` config for non-object zone entries. @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Reserved the complete admin namespace at the publisher-fallback boundary. Percent-encoded separators (`/_ts/admin%2Fec`, `%2f`, and double-encoded forms) matched the `^/_ts/admin` Basic-auth handler but escaped the literal-slash namespace check, so an authenticated request fell through to publisher fallback and forwarded its `Authorization` header and body to the publisher origin. The reservation now spans the whole `/_ts/admin` prefix plus the retired `/admin/keys` aliases — including trailing, descendant, and encoded-separator forms — evaluated on both the raw and percent-decoded path, and applies to every adapter. - Validate synthetic ID format on inbound values from the `x-synthetic-id` header and `synthetic_id` cookie; values that do not match the expected format (`64-hex-hmac.6-alphanumeric-suffix`) are discarded and a fresh ID is generated rather than forwarded to response headers, cookies, or third-party APIs ### Fixed diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 2fe123fcb..bacb7aa36 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2206,10 +2206,10 @@ impl Settings { /// Update [`ADMIN_ENDPOINTS`](Self::ADMIN_ENDPOINTS) when adding new /// admin routes to `crates/trusted-server-adapter-fastly/src/app.rs`. /// - /// The `/_ts/admin/ec/{id}` entry is the canonical router pattern. Handler - /// coverage is checked against representative concrete EC IDs via - /// [`admin_auth_probes`](Self::admin_auth_probes), while validation errors - /// continue to report this operator-facing route template. + /// The `/_ts/admin/ec/{id}` entry is the canonical router pattern. Its + /// coverage is checked via [`admin_auth_probes`](Self::admin_auth_probes), + /// while validation errors continue to report this operator-facing route + /// template. pub(crate) const ADMIN_ENDPOINTS: &[&str] = &[ "/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate", @@ -2218,12 +2218,22 @@ impl Settings { "/_ts/admin/eids", ]; + /// Probes that establish handler coverage for the dynamic + /// `/_ts/admin/ec/{id}` route. + /// + /// Coverage cannot be sampled: the router accepts any single segment after + /// `/_ts/admin/ec/` and basic auth runs on the raw path before routing, so + /// a handler that matches only some ID shapes leaves the rest of the route + /// surface — including malformed IDs, which still reach the admin handler — + /// unauthenticated at configuration time and fail-closed at runtime. + /// + /// Both probes must match the same configuration for the route to count as + /// covered. The bare prefix rejects handlers anchored to specific ID + /// shapes; the concrete ID rejects handlers anchored to the prefix itself + /// (`^/_ts/admin/ec/$`). Together they admit only prefix-level matchers + /// such as `^/_ts/admin` or `^/_ts/admin/ec/`. const ADMIN_EC_ID_AUTH_PROBES: [&str; 2] = [ - concat!( - "/_ts/admin/ec/", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - ".abc123", - ), + "/_ts/admin/ec/", concat!( "/_ts/admin/ec/", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", @@ -2291,26 +2301,19 @@ impl Settings { })) } + /// Rejects placeholder and well-known weak handler passwords. + /// + /// Applies to every handler rather than to handlers inferred to cover an + /// admin endpoint: handler selection is first-match-wins over operator + /// regexes, so a narrow handler can shadow the admin namespace for paths no + /// probe enumerates. Handlers are Trusted Server's own basic-auth gates, so + /// a placeholder password is never valid on any of them. fn validate_admin_handler_passwords(&self) -> Result<(), Report> { for handler in &self.handlers { - let covers_admin = - Self::ADMIN_ENDPOINTS - .iter() - .try_fold(false, |covers_any_endpoint, path| { - Self::admin_auth_probes(path).iter().try_fold( - covers_any_endpoint, - |covers_any_probe, probe| { - handler - .matches_path(probe) - .map(|matches| covers_any_probe || matches) - }, - ) - })?; - - if covers_admin && is_admin_placeholder_password(handler.password.expose()) { + if is_admin_placeholder_password(handler.password.expose()) { return Err(Report::new(TrustedServerError::Configuration { message: format!( - "Admin handler `{}` uses a placeholder password; configure a strong secret", + "Handler `{}` uses a placeholder password; configure a strong secret", handler.path ), })); @@ -5024,7 +5027,65 @@ origin_host_header_overide = "www.example.com""#, } #[test] - fn from_toml_rejects_placeholder_password_for_concrete_admin_ec_handler() { + fn from_toml_rejects_placeholder_password_on_shadowing_admin_handler() { + // Handler selection is first-match-wins, so a narrow handler placed + // ahead of the admin matcher governs the EC IDs it matches. No probe + // enumerates those IDs, so the placeholder check cannot be limited to + // handlers inferred to cover an admin endpoint. + let toml_str = crate_test_settings_str().replace( + r#"path = "^/_ts/admin" + username = "admin" + password = "admin-pass""#, + r#"path = "^/_ts/admin/ec/[a-f0-9]{64}[.]zzzzzz$" + username = "admin" + password = "change-me-admin-password" + + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "strong-test-password""#, + ); + + let error = Settings::from_toml(&toml_str) + .expect_err("should reject placeholder password on shadowing admin handler"); + let message = format!("{error:?}"); + assert!( + message.contains("placeholder password"), + "should identify the placeholder handler password, got: {message}" + ); + } + + #[test] + fn from_toml_rejects_weak_password_on_non_admin_handler() { + let toml_str = crate_test_settings_str().replace( + r#"path = "^/_ts/admin" + username = "admin" + password = "admin-pass""#, + r#"path = "^/_ts/admin" + username = "admin" + password = "strong-test-password" + + [[handlers]] + path = "^/private" + username = "admin" + password = "changeme""#, + ); + + let error = Settings::from_toml(&toml_str) + .expect_err("should reject a weak password on any handler"); + let message = format!("{error:?}"); + assert!( + message.contains("placeholder password"), + "should identify the weak handler password, got: {message}" + ); + } + + #[test] + fn from_toml_rejects_sampled_id_only_dynamic_admin_ec_auth_coverage() { + // A handler anchored to the full EC ID grammar still leaves the rest of + // the route surface (malformed IDs, which the router accepts and the + // admin handler rejects with 400) unauthenticated, so coverage must not + // be inferred from ID-shaped samples. let toml_str = crate_test_settings_str().replace( r#"path = "^/_ts/admin" username = "admin" @@ -5036,16 +5097,62 @@ origin_host_header_overide = "www.example.com""#, [[handlers]] path = "^/_ts/admin/ec/[a-f0-9]{64}[.][A-Za-z0-9]{6}$" username = "admin" - password = "change-me-admin-password""#, + password = "strong-test-password""#, ); let error = Settings::from_toml(&toml_str) - .expect_err("should reject placeholder password on concrete EC handler"); + .expect_err("should reject ID-sampled dynamic EC auth coverage"); let message = format!("{error:?}"); assert!( - message.contains("placeholder password"), - "should identify the placeholder admin password, got: {message}" + message.contains("/_ts/admin/ec/{id}"), + "should identify the dynamic EC route as uncovered, got: {message}" + ); + } + + #[test] + fn from_toml_rejects_prefix_anchored_admin_ec_auth_coverage() { + // `^/_ts/admin/ec/$` matches the prefix probe but no actual lookup. + let toml_str = crate_test_settings_str().replace( + r#"path = "^/_ts/admin" + username = "admin" + password = "admin-pass""#, + r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" + username = "admin" + password = "strong-test-password" + + [[handlers]] + path = "^/_ts/admin/ec/$" + username = "admin" + password = "strong-test-password""#, ); + + let error = Settings::from_toml(&toml_str) + .expect_err("should reject prefix-anchored dynamic EC auth coverage"); + let message = format!("{error:?}"); + assert!( + message.contains("/_ts/admin/ec/{id}"), + "should identify the dynamic EC route as uncovered, got: {message}" + ); + } + + #[test] + fn from_toml_accepts_prefix_matcher_admin_ec_auth_coverage() { + let toml_str = crate_test_settings_str().replace( + r#"path = "^/_ts/admin" + username = "admin" + password = "admin-pass""#, + r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" + username = "admin" + password = "strong-test-password" + + [[handlers]] + path = "^/_ts/admin/ec/" + username = "admin" + password = "strong-test-password""#, + ); + + Settings::from_toml(&toml_str) + .expect("should accept a prefix-level matcher for the dynamic EC route"); } #[test] diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index ddb6544ce..905df9e60 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -645,6 +645,23 @@ path = "^/api/v[0-9]+/private" # /api/v1/private, /api/v2/private **Validation**: Application startup fails if regex is invalid. +::: warning Admin coverage and passwords are validated at startup + +Startup fails when no handler covers an admin route. The dynamic +`/_ts/admin/ec/{id}` route accepts any segment after `/_ts/admin/ec/`, and +Basic Auth runs on the raw path before routing, so coverage cannot be inferred +from ID-shaped samples: a pattern such as +`^/_ts/admin/ec/[a-f0-9]{64}[.][A-Za-z0-9]{6}$` is rejected. Use a prefix-level +matcher (`^/_ts/admin`, or `^/_ts/admin/ec/` alongside the other admin +patterns). + +Startup also fails when any handler — admin or not — uses a placeholder or +well-known weak password (`changeme`, `password`, `admin`, or a +`replace-with-…` template value). Handler selection is first-match-wins, so a +narrow handler ahead of the admin pattern governs the paths it matches. + +::: + ::: warning Scope patterns to the paths you mean Handler patterns are matched against the full request path, so a broad pattern From 9a1cba956e335af0d52864cdc39fb12eeab00d10 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 23:37:21 +0530 Subject: [PATCH 377/395] Mark diagnostics responses terminal-private `finalize_response` stamped `Cache-Control: private, no-store` directly and left no marker behind, so the adapter's terminal guard had nothing to re-enforce from. An active decision that sets no new cookie also escapes the `Set-Cookie` privacy net, which left a late `RequestFilterEffects` mutation such as `Cache-Control: public` free to make request-scoped diagnostics HTML eligible for a shared cache. Use `enforce_synthesized_html_cache_privacy`, which applies the same policy and marks the response `TerminalPrivateResponse`. It also drops the origin validators and expiry metadata, matching every other synthesized-HTML path. --- .../trusted-server-adapter-fastly/src/main.rs | 88 +++++++++++++++++++ .../src/integrations/gpt_diagnostics.rs | 88 +++++++++++++++++-- 2 files changed, 167 insertions(+), 9 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index be4feb6f1..06f5f1aa7 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -607,6 +607,94 @@ mod tests { assert!(response.headers().get("etag").is_none()); } + fn diagnostics_settings() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + + [integrations.gpt_diagnostics] + enabled = true + "#, + ) + .expect("should parse diagnostics settings") + } + + #[test] + fn late_filter_effects_cannot_make_an_active_diagnostics_response_public() { + // The narrowest hole: an established diagnostics session sets no new cookie, so + // the `Set-Cookie` privacy net never fires, and before this the decision only + // stamped `Cache-Control` without leaving a marker for the terminal guard. + let mut request = edgezero_core::http::request_builder() + .method(fastly::http::Method::GET) + .uri("https://test-publisher.com/article") + .header("sec-fetch-dest", "document") + .header("cookie", "__Host-ts-console=1") + .body(EdgeBody::empty()) + .expect("should build request"); + let decision = trusted_server_core::integrations::gpt_diagnostics::prepare_request( + &diagnostics_settings(), + &mut request, + ) + .expect("should prepare the diagnostics decision"); + assert!( + decision.active(), + "the session cookie should activate diagnostics" + ); + + let mut response = response_builder() + .header("cache-control", "public, max-age=600") + .body(EdgeBody::empty()) + .expect("should build response"); + trusted_server_core::integrations::gpt_diagnostics::finalize_response( + &decision, + &mut response, + ); + + let effects = RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![ + HeaderMutation::set("cache-control", "public, s-maxage=3600"), + HeaderMutation::set("surrogate-control", "max-age=3600"), + ], + }; + + apply_terminal_response_effects(&mut response, Some(&effects)); + + assert!( + response.headers().get("set-cookie").is_none(), + "the case under test is the one with no Set-Cookie to protect it" + ); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("private, no-store"), + "request-scoped diagnostics HTML must never become shared-cacheable" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip CDN cache directives a late filter added" + ); + } + #[test] fn terminal_response_preserves_unmarked_origin_private_policy() { let mut response = response_builder() diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 487643142..94e4607cd 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -14,7 +14,7 @@ use edgezero_core::body::Body as EdgeBody; use crate::error::TrustedServerError; use crate::http_util::is_navigation_request; -use crate::response_privacy::CDN_CACHE_HEADERS; +use crate::response_privacy::enforce_synthesized_html_cache_privacy; use crate::settings::{IntegrationConfig, Settings}; use crate::tsjs; @@ -334,13 +334,12 @@ pub fn finalize_response( } if decision.requires_private_no_store() { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); - for name in CDN_CACHE_HEADERS { - response.headers_mut().remove(*name); - } + // Marks the response terminal-private as well as stamping it. Stamping alone + // left the policy at the mercy of whatever ran later: a late + // `RequestFilterEffects` mutation such as `Cache-Control: public` replaced it, + // and the adapter's terminal guard had no marker to re-enforce from, so + // request-scoped diagnostics HTML became shared-cacheable. + enforce_synthesized_html_cache_privacy(response); } } @@ -578,6 +577,8 @@ mod tests { let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); let mut response = Response::builder() .header(header::CACHE_CONTROL, "public, max-age=60") + .header(header::ETAG, "\"origin\"") + .header(header::LAST_MODIFIED, "Wed, 12 Aug 2026 00:00:00 GMT") .header("surrogate-control", "max-age=60") .header("fastly-surrogate-control", "max-age=60") .header("cloudflare-cdn-cache-control", "public, max-age=60") @@ -588,7 +589,8 @@ mod tests { assert_eq!( response.headers()[header::CACHE_CONTROL], - "private, no-store" + "private, no-store", + "should stamp diagnostics responses non-storable" ); assert_eq!(response.headers()[header::SET_COOKIE], SET_CONSOLE_COOKIE); assert!(!response.headers().contains_key("surrogate-control")); @@ -598,6 +600,74 @@ mod tests { .headers() .contains_key("cloudflare-cdn-cache-control") ); + assert!( + !response.headers().contains_key(header::ETAG), + "should drop the origin validator with the shared-cache policy" + ); + assert!( + !response.headers().contains_key(header::LAST_MODIFIED), + "should drop the origin validator with the shared-cache policy" + ); + } + + #[test] + fn an_active_no_cookie_action_response_is_marked_terminal_private() { + // The session-cookie activation path: active, but nothing new to set. Stamping + // `Cache-Control` alone left this response defenceless against a later mutation, + // because the adapter's terminal guard keys on the marker, not on the stamp, and + // the `Set-Cookie` privacy net never sees a response that sets no cookie. + let mut request = navigation("https://publisher.example/", Some("__Host-ts-console=1")); + let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + assert!(decision.active(), "the session cookie should activate"); + assert_eq!( + decision.cookie_action, + GptDiagnosticsCookieAction::None, + "an already-established session sets no new cookie" + ); + let mut response = Response::builder() + .body(EdgeBody::empty()) + .expect("should build response"); + + finalize_response(&decision, &mut response); + + assert!( + !response.headers().contains_key(header::SET_COOKIE), + "should not set a cookie for an established session" + ); + assert!( + response + .extensions() + .get::() + .is_some(), + "should mark request-scoped diagnostics HTML for terminal re-enforcement" + ); + } + + #[test] + fn an_inactive_decision_leaves_the_origin_cache_policy_alone() { + let mut request = navigation("https://publisher.example/", None); + let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + assert!(!decision.requires_private_no_store()); + let mut response = Response::builder() + .header(header::CACHE_CONTROL, "public, max-age=60") + .header(header::ETAG, "\"origin\"") + .body(EdgeBody::empty()) + .expect("should build response"); + + finalize_response(&decision, &mut response); + + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "public, max-age=60", + "should not downgrade a response the integration did not touch" + ); + assert!( + response + .extensions() + .get::() + .is_none(), + "should not mark an untouched response terminal-private" + ); } #[test] From 82db770297e9455c365d937fb60d578dd8d2c5f6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 19 Aug 2026 23:37:44 +0530 Subject: [PATCH 378/395] Anchor the shared seam to the parsed body end The seam was spliced at the last `` sequence found in the transform's output bytes. That position is not knowable from bytes: a document whose only `` lives in `` took the seam inside the string literal, where the payload's own `` terminated the publisher's script, and a `` in trailing comment data outranked the real closing tag, burying the bids in a comment. Both corrupted the response served cold and the template every warm reader received. Emit `TEMPLATE_SEAM_PLACEHOLDER` from the rewriter's structural body end-tag handler instead, reusing the document-end fallback for documents with no close, and substitute the seam or this reader's bids there. The placeholder is deliberately distinct from the seam, so a publisher document containing the seam bytes still receives correctly positioned bids; a collision with the placeholder itself refuses the substitution rather than guessing. Also refuse to store a document that delivers a response-bound CSP nonce in its own markup. The eligibility gate only inspected response headers, so an origin could deliver the policy in `` and have that nonce replayed to every later reader. The observation is structural, made by the parser, because a byte scan cannot tell a `nonce` attribute from the same word in a script. Drop the source-comment neutralizer with it: nothing used it, and rewriting publisher comments to protect a marker changes publisher content bytes. --- .../benches/html_processor_bench.rs | 1 + .../trusted-server-core/src/html_processor.rs | 212 +++++++- crates/trusted-server-core/src/publisher.rs | 465 +++++++++++++++--- 3 files changed, 600 insertions(+), 78 deletions(-) diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index de968301c..19aa0b82f 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -7,6 +7,7 @@ use trusted_server_core::streaming_processor::StreamProcessor as _; fn make_config() -> HtmlProcessorConfig { HtmlProcessorConfig { + csp_nonce_observed: None, origin_host: "origin.bench.example.com".to_string(), request_host: "proxy.bench.example.com".to_string(), request_scheme: "https".to_string(), diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index a5e4c9d5b..9f17f64bc 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use lol_html::{ - EndTagHandler, Settings as RewriterSettings, doc_comments, element, end, + EndTagHandler, Settings as RewriterSettings, element, end, html_content::{ContentType, EndTag}, text, }; @@ -204,6 +204,11 @@ pub struct HtmlProcessorConfig { pub body_close: BodyCloseInjection, /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. pub suppress_datadome_client_side_tag: bool, + /// Set when the document delivers a response-bound CSP nonce in its own markup. + /// + /// `None` on every path that cannot store a shared template, so an ordinary inline + /// request does not pay for handlers whose only consumer is the template-cache gate. + pub csp_nonce_observed: Option>, } impl HtmlProcessorConfig { @@ -227,6 +232,7 @@ impl HtmlProcessorConfig { gpt_diagnostics: None, body_close: BodyCloseInjection::None, suppress_datadome_client_side_tag: false, + csp_nonce_observed: None, } } @@ -266,6 +272,16 @@ impl HtmlProcessorConfig { self } + /// Watch the document for a response-bound CSP nonce delivered in its own markup. + /// + /// Pass `Some` only when the completed transform may be stored as a shared template; + /// nothing else reads the observation. + #[must_use] + pub fn with_csp_nonce_observer(mut self, observed: Option>) -> Self { + self.csp_nonce_observed = observed; + self + } + /// Attach the request-scoped `DataDome` client-tag suppression decision. #[must_use] pub fn with_datadome_client_tag_suppression(mut self, suppress: bool) -> Self { @@ -360,25 +376,11 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let ad_bids_state = config.ad_bids_state.clone(); let gpt_diagnostics = config.gpt_diagnostics.clone(); - // A publisher can legitimately emit the same inert comment text as the reserved - // template-cache seam, including after ``. Neutralize source comments while they are - // parsed; markup injected by the body end-tag handler is output, not reparsed, so - // the transform-owned marker remains the only exact copy. + // No source-comment neutralization here: rewriting a publisher comment that happens + // to match the reserved marker would change publisher content bytes. Collisions are + // detected on the completed transform instead, where the response can be refused + // outright rather than silently edited. let mut document_content_handlers = Vec::new(); - if let BodyCloseInjection::Marker(marker) = &body_close - && let Some(reserved) = marker - .strip_prefix("")) - { - let reserved = reserved.to_string(); - let escaped = format!("x{reserved}"); - document_content_handlers.push(doc_comments!(move |comment| { - if comment.text() == reserved { - comment.set_text(&escaped)?; - } - Ok(()) - })); - } if let BodyCloseInjection::Marker(marker) = &body_close { let marker = marker.clone(); let injected_bids = Arc::clone(&injected_bids); @@ -712,6 +714,36 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso }), ]; + // A response-bound nonce is only safe for the response that carried it, and the + // response-header gate cannot see one the origin delivered in the markup instead. + // Observed structurally rather than by scanning the output bytes, which cannot tell a + // `nonce` attribute from the same word inside a script. + if let Some(observed) = config.csp_nonce_observed.clone() { + let meta_observed = Arc::clone(&observed); + element_content_handlers.push(element!("meta[http-equiv][content]", move |el| { + let delivers_csp = el.get_attribute("http-equiv").is_some_and(|equiv| { + matches!( + equiv.trim().to_ascii_lowercase().as_str(), + "content-security-policy" | "content-security-policy-report-only" + ) + }); + if delivers_csp + && el + .get_attribute("content") + .is_some_and(|policy| policy.to_ascii_lowercase().contains("'nonce-")) + { + meta_observed.store(true, Ordering::SeqCst); + } + Ok(()) + })); + // Nonce attributes are rejected on their own: a document carrying them is written + // for a per-response policy whether or not the policy itself reached this scan. + element_content_handlers.push(element!("[nonce]", move |_el| { + observed.store(true, Ordering::SeqCst); + Ok(()) + })); + } + for script_rewriter in script_rewriters { let selector = script_rewriter.selector(); let rewriter = script_rewriter.clone(); @@ -785,6 +817,7 @@ mod tests { fn create_test_config() -> HtmlProcessorConfig { HtmlProcessorConfig { + csp_nonce_observed: None, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_owned(), request_host: "test.example.com".to_owned(), @@ -1687,6 +1720,7 @@ mod tests { #[test] fn injects_ad_slots_at_head_open() { let config = HtmlProcessorConfig { + csp_nonce_observed: None, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1764,6 +1798,7 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + csp_nonce_observed: None, body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1802,6 +1837,7 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + csp_nonce_observed: None, body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1841,6 +1877,7 @@ mod tests { let request_host = "proxy.test-publisher.example.com"; let config = HtmlProcessorConfig { + csp_nonce_observed: None, body_close: BodyCloseInjection::None, origin_host: "origin.test-publisher.example.com".to_string(), request_host: request_host.to_string(), @@ -1894,6 +1931,7 @@ mod tests { // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + csp_nonce_observed: None, body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1925,6 +1963,7 @@ mod tests { // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + csp_nonce_observed: None, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1947,10 +1986,145 @@ mod tests { ); } + fn marker_mode_config(marker: &str, observer: Option>) -> HtmlProcessorConfig { + HtmlProcessorConfig { + csp_nonce_observed: observer, + body_close: BodyCloseInjection::Marker(marker.to_string()), + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), + max_buffered_body_bytes: 16 * 1024 * 1024, + gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, + } + } + + fn render_marker_mode(marker: &str, source: &str) -> String { + let mut processor = create_html_processor(marker_mode_config(marker, None)); + let output = processor + .process_chunk(source.as_bytes(), true) + .expect("should process the document"); + String::from_utf8(output).expect("output should be utf8") + } + + #[test] + fn marker_mode_ignores_a_body_close_written_in_script_data() { + // A reverse byte search for `` picks this string literal, because the + // document has no structural close at all. Splicing a `` inside the publisher's script and corrupts the document — + // and, once stored, every warm reader of it. Only the parser can tell the + // difference, so the parser places the marker. + const MARKER: &str = ""; + let source = + r#"

a

"#; + + let html = render_marker_mode(MARKER, source); + + assert!( + html.contains(r#"const marker = "";"#), + "should leave the publisher's script data byte for byte: {html}" + ); + assert_eq!( + html.matches(MARKER).count(), + 1, + "should emit exactly one transform-owned marker: {html}" + ); + assert!( + html.ends_with(MARKER), + "a document with no structural body close takes the terminal marker: {html}" + ); + } + + #[test] + fn marker_mode_prefers_the_structural_body_close_over_trailing_comment_data() { + // A reverse byte search takes the *last* `` sequence, which here lives in + // trailing comment data, so the marker landed after the document's real end. + const MARKER: &str = ""; + let source = "

a

"; + + let html = render_marker_mode(MARKER, source); + + assert!( + html.contains(&format!("

a

{MARKER}")), + "should place the marker at the structural body close: {html}" + ); + assert!( + html.contains(""), + "should leave the publisher's trailing comment untouched: {html}" + ); + assert_eq!( + html.matches(MARKER).count(), + 1, + "should emit exactly one transform-owned marker: {html}" + ); + } + + #[test] + fn a_nonce_bearing_meta_policy_is_observed() { + let observed = Arc::new(AtomicBool::new(false)); + let mut processor = + create_html_processor(marker_mode_config("", Some(Arc::clone(&observed)))); + + processor + .process_chunk( + br#"a"#, + true, + ) + .expect("should process the document"); + + assert!( + observed.load(Ordering::SeqCst), + "a policy delivered in markup is invisible to the response-header gate" + ); + } + + #[test] + fn a_nonce_attribute_is_observed() { + let observed = Arc::new(AtomicBool::new(false)); + let mut processor = + create_html_processor(marker_mode_config("", Some(Arc::clone(&observed)))); + + processor + .process_chunk( + b"a", + true, + ) + .expect("should process the document"); + + assert!( + observed.load(Ordering::SeqCst), + "a document written for a per-response nonce must not be shared" + ); + } + + #[test] + fn the_word_nonce_in_script_text_is_not_observed() { + // The reason this is structural rather than a byte scan over the output. + let observed = Arc::new(AtomicBool::new(false)); + let mut processor = + create_html_processor(marker_mode_config("", Some(Arc::clone(&observed)))); + + processor + .process_chunk( + br#"a"#, + true, + ) + .expect("should process the document"); + + assert!( + !observed.load(Ordering::SeqCst), + "ordinary script text must not cost a cacheable page its shared template" + ); + } + #[test] fn bodyless_marker_mode_emits_an_owned_terminal_seam_even_after_source_bytes() { const MARKER: &str = ""; let config = HtmlProcessorConfig { + csp_nonce_observed: None, body_close: BodyCloseInjection::Marker(MARKER.to_string()), origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index b62851460..cc559975a 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -20,6 +20,7 @@ use std::borrow::Cow; use std::io::Write; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime}; @@ -590,6 +591,8 @@ struct ProcessResponseParams<'a> { Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, /// See [`HtmlStreamProcessorParams::shared_template_authorized`]. shared_template_authorized: bool, + /// See [`HtmlStreamProcessorParams::csp_nonce_observed`]. + csp_nonce_observed: Option<&'a Arc>, } struct PublisherBodyProcessor { @@ -617,6 +620,7 @@ impl PublisherBodyProcessor { suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), shared_template_authorized: params.template_cache_key.is_some(), + csp_nonce_observed: params.csp_nonce_observed.clone(), })?) } else if is_rsc_flight { Box::new(RscFlightUrlRewriter::new( @@ -697,6 +701,7 @@ fn process_response_streaming( suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.cloned(), shared_template_authorized: params.shared_template_authorized, + csp_nonce_observed: params.csp_nonce_observed.cloned(), })?; StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) @@ -1196,6 +1201,8 @@ struct HtmlStreamProcessorParams<'a> { /// Carried rather than re-derived so both seams see the same answer. See /// [`effective_assembly_mode`]. shared_template_authorized: bool, + /// Where the transform records a response-bound CSP nonce, when one matters. + csp_nonce_observed: Option>, } /// The diagnostics decision the template may carry. @@ -1238,6 +1245,21 @@ pub(crate) fn template_gpt_diagnostics( /// any escaping question at the seam. pub const AD_ASSEMBLY_SEAM: &str = ""; +/// Transform-owned stand-in for the seam, emitted at the document's structural body end. +/// +/// Deliberately *not* [`AD_ASSEMBLY_SEAM`]. The payload that ends up in the seam is not +/// known until the completed transform has been checked for publisher collisions, and the +/// position is not knowable from the output bytes: a reverse search for `` selects +/// a string literal in `` when the document has +/// no real close, and prefers a `` sequence in trailing comment data over the real +/// closing tag. Only the parser knows which one is structural, so the parser marks the +/// spot and the substitution below fills it in. +/// +/// Keeping it distinct from [`AD_ASSEMBLY_SEAM`] is what lets a publisher document that +/// contains the seam bytes still receive correctly positioned bids: that collision +/// revokes the shared reservation without disturbing this placeholder. +pub(crate) const TEMPLATE_SEAM_PLACEHOLDER: &str = ""; + /// The mode the operator asked for, before availability is taken into account. /// /// Spelled once, because the mode has to mean the same thing at the cache key, at the @@ -1302,9 +1324,11 @@ fn effective_assembly_mode(settings: &Settings, shared_template_authorized: bool /// two independent decisions: once [`template_ad_slots_script`] stopped emitting a /// head script under a shared mode, body-close injection stopped with it. /// -/// `Esi` emits nothing here. Its inert marker is inserted only after the completed -/// transform has been checked for publisher collisions and ESI directives, so TS never -/// has to guess which identical marker belongs to the publisher. +/// `Esi` emits [`TEMPLATE_SEAM_PLACEHOLDER`], not the seam itself. What goes into the +/// seam is still decided after the completed transform has been checked for publisher +/// collisions and ESI directives — but *where* it goes has to be decided here, by the +/// parser, because the output bytes cannot distinguish a structural `` from one +/// written inside a script string or a trailing comment. pub(crate) fn body_close_injection( mode: AssemblyMode, head_script_present: bool, @@ -1318,7 +1342,7 @@ pub(crate) fn body_close_injection( BodyCloseInjection::None } } - AssemblyMode::Esi => BodyCloseInjection::None, + AssemblyMode::Esi => BodyCloseInjection::Marker(TEMPLATE_SEAM_PLACEHOLDER.to_string()), } } @@ -1340,10 +1364,18 @@ fn create_html_stream_processor( let gpt_diagnostics = template_gpt_diagnostics(assembly_mode, params.gpt_diagnostics); + // Only a response that can be stored has a consumer for the observation, so the + // handlers are not registered for ordinary inline traffic. + let csp_nonce_observed = params + .shared_template_authorized + .then_some(params.csp_nonce_observed) + .flatten(); + let config = config .with_ad_state(params.ad_slots_script, params.ad_bids_state) .with_gpt_diagnostics(gpt_diagnostics) .with_body_close(body_close) + .with_csp_nonce_observer(csp_nonce_observed) .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); Ok(create_html_processor(config)) @@ -1505,6 +1537,12 @@ pub struct OwnedProcessResponseParams { /// Request-scoped conditional diagnostics delivery decision. pub(crate) gpt_diagnostics: Option, + /// Set by the transform when the document carries a response-bound CSP nonce. + /// + /// `None` wherever no transform runs. Recorded by the HTML parser rather than + /// rescanned from the output, which cannot tell a `nonce` attribute from the same + /// word inside a script. + pub(crate) csp_nonce_observed: Option>, } /// Response-authorized template cache insert inputs. The key is built before origin lookup; the @@ -1620,32 +1658,43 @@ pub async fn buffer_publisher_response_async( // request cannot store twice, which would leave nothing for assembly to gate // on. let was_authorized = params.template_cache_key.is_some(); - let contains_publisher_marker = was_authorized - && bytes - .windows(AD_ASSEMBLY_SEAM.len()) - .any(|window| window == AD_ASSEMBLY_SEAM.as_bytes()); - let contains_publisher_esi = was_authorized && contains_publisher_esi_directive(&bytes); - let bypasses_shared_template = contains_publisher_marker || contains_publisher_esi; - if bypasses_shared_template { - log::warn!( - "template_cache bypass: transformed response contains publisher-authored {}", - if contains_publisher_marker { - "seam bytes" - } else { - "ESI" - } - ); + let shared_bypass_reason = was_authorized + .then(|| shared_template_bypass_reason(&bytes, params.csp_nonce_observed.as_ref())) + .flatten(); + if let Some(reason) = shared_bypass_reason { + log::warn!("template_cache bypass: transformed response {reason}"); params.template_cache_key.take(); } - let shared_response_authorized = was_authorized && !bypasses_shared_template; - let bytes = if shared_response_authorized { - insert_before_body_close(bytes, AD_ASSEMBLY_SEAM.as_bytes()) - } else if was_authorized { - let seam = seam_script_for(¶ms); - insert_before_body_close(bytes, seam.as_bytes()) + let mut shared_response_authorized = was_authorized && shared_bypass_reason.is_none(); + // The parser marked the structural body end during the transform; this puts the + // right payload there. A shared response gets the inert seam every reader will + // split on, a bypassed one gets this reader's bids directly. + let bytes = if was_authorized { + let payload: Cow<'_, str> = if shared_response_authorized { + Cow::Borrowed(AD_ASSEMBLY_SEAM) + } else { + Cow::Owned(seam_script_for(¶ms)) + }; + match replace_seam_placeholder(bytes, payload.as_bytes()) { + Ok(bytes) => bytes, + Err((bytes, error)) => { + // Publisher bytes collided with the transform's own placeholder, so + // there is no position TS can claim. Serving the document untouched + // costs this page its bids; guessing a position corrupts it and, if + // stored, every later reader of it too. + log::warn!( + "template_cache bypass: {error}; serving the transformed document \ + without a seam" + ); + params.template_cache_key.take(); + shared_response_authorized = false; + bytes + } + } } else { bytes }; + let bypasses_shared_template = was_authorized && !shared_response_authorized; // Validate before the store, not after. // // `assemble_if_shared` does the same split and would reject a malformed @@ -1895,6 +1944,7 @@ fn build_template_assembly_params( ad_bids_state: AdBidsState, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + csp_nonce_observed: None, // Already stored; storing again on a hit would be pointless work. template_cache_key: None, seam_ad_slots: None, @@ -1926,32 +1976,79 @@ fn build_template_assembly_params( /// with either no bids or a visible marker in it. fn split_template_at_seam(template: &[u8]) -> Result<(&[u8], &[u8]), SeamError> { let marker = AD_ASSEMBLY_SEAM.as_bytes(); - let mut found = template - .windows(marker.len()) + let at = only_occurrence(template, marker)?; + Ok((&template[..at], &template[at + marker.len()..])) +} + +/// Offset of the one and only occurrence of `needle`. +/// +/// # Errors +/// +/// Returns [`SeamError::Missing`] when `needle` is absent and [`SeamError::Repeated`] +/// when it appears more than once — which, for a transform-owned marker, means publisher +/// bytes collided with it and no occurrence can be claimed as ours. +fn only_occurrence(haystack: &[u8], needle: &[u8]) -> Result { + let mut found = haystack + .windows(needle.len()) .enumerate() - .filter(|(_, w)| *w == marker) + .filter(|(_, window)| *window == needle) .map(|(at, _)| at); let at = found.next().ok_or(SeamError::Missing)?; if found.next().is_some() { return Err(SeamError::Repeated); } - Ok((&template[..at], &template[at + marker.len()..])) + Ok(at) } -/// Insert a reader payload at the document's final body close, or append it to a fragment. +/// Why the completed transform must not be stored as a shared template, if it must not. /// -/// The final case-insensitive close matches the streaming pipeline's body-tail convention -/// while avoiding an earlier `""` string in script data. -fn insert_before_body_close(mut document: Vec, payload: &[u8]) -> Vec { - if payload.is_empty() { - return document; - } - let insertion_at = document - .windows(BODY_CLOSE_PREFIX.len()) - .rposition(|window| window.eq_ignore_ascii_case(BODY_CLOSE_PREFIX)) - .unwrap_or(document.len()); - document.splice(insertion_at..insertion_at, payload.iter().copied()); - document +/// Every one of these is invisible before the transform runs. Publisher-authored seam +/// bytes and ESI survive it, and a CSP nonce the origin delivered in a `` policy or +/// on its own elements never appears in a response header, which is all the eligibility +/// gate gets to inspect. +fn shared_template_bypass_reason( + bytes: &[u8], + csp_nonce_observed: Option<&Arc>, +) -> Option<&'static str> { + if bytes + .windows(AD_ASSEMBLY_SEAM.len()) + .any(|window| window == AD_ASSEMBLY_SEAM.as_bytes()) + { + return Some("contains publisher-authored seam bytes"); + } + if contains_publisher_esi_directive(bytes) { + return Some("contains publisher-authored ESI"); + } + if csp_nonce_observed.is_some_and(|observed| observed.load(Ordering::SeqCst)) { + return Some("delivers a response-bound CSP nonce in its own markup"); + } + None +} + +/// Substitute `payload` for the transform-owned [`TEMPLATE_SEAM_PLACEHOLDER`]. +/// +/// The placeholder was written by the HTML parser at the document's structural body end, +/// or appended at document end when the document has no body close at all — so this +/// carries the parser's answer rather than re-deriving it from bytes that cannot express +/// it. +/// +/// # Errors +/// +/// Returns the untouched document together with the reason when the placeholder is absent +/// or appears more than once. Both mean publisher bytes collided with it, and inserting at +/// a guessed position would corrupt the document and the template stored from it. +fn replace_seam_placeholder( + mut document: Vec, + payload: &[u8], +) -> Result, (Vec, SeamError)> { + let placeholder = TEMPLATE_SEAM_PLACEHOLDER.as_bytes(); + match only_occurrence(&document, placeholder) { + Ok(at) => { + document.splice(at..at + placeholder.len(), payload.iter().copied()); + Ok(document) + } + Err(error) => Err((document, error)), + } } /// Why a template could not be split at its seam. @@ -2610,6 +2707,7 @@ pub fn stream_publisher_body( suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.as_ref(), shared_template_authorized: params.template_cache_key.is_some(), + csp_nonce_observed: params.csp_nonce_observed.as_ref(), }; let input_compression = Compression::from_content_encoding(¶ms.content_encoding); let output_compression = if params.template_cache_key.is_some() { @@ -2711,6 +2809,7 @@ pub async fn stream_publisher_body_async( suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), shared_template_authorized: params.template_cache_key.is_some(), + csp_nonce_observed: params.csp_nonce_observed.clone(), }) { Ok(processor) => processor, Err(err) => { @@ -4548,6 +4647,10 @@ pub async fn handle_publisher_request( response, body, params: Box::new(OwnedProcessResponseParams { + // The transform writes here; the post-transform gate reads it. Always + // present on the live path so no future caller has to remember to + // supply it — the handlers themselves stay gated on authorization. + csp_nonce_observed: Some(Arc::new(AtomicBool::new(false))), template_cache_key, seam_ad_slots, policy_headers, @@ -6616,6 +6719,7 @@ mod tests { content_encoding: &str, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -6894,6 +6998,7 @@ mod tests { }))); let config = HtmlProcessorConfig { + csp_nonce_observed: None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -6936,12 +7041,15 @@ mod tests { (4, ""), "the readable seam and its cache schema must move together" ); - assert!( - matches!( - body_close_injection(AssemblyMode::Esi, false), - BodyCloseInjection::None - ), - "should insert the shared seam only after checking completed publisher bytes" + assert_eq!( + body_close_injection(AssemblyMode::Esi, false), + BodyCloseInjection::Marker(TEMPLATE_SEAM_PLACEHOLDER.to_string()), + "the seam's position must come from the parser, never from a byte search" + ); + assert_ne!( + TEMPLATE_SEAM_PLACEHOLDER, AD_ASSEMBLY_SEAM, + "a publisher document carrying the seam bytes must still receive \ + correctly positioned bids, which a shared marker would make impossible" ); } @@ -8544,26 +8652,75 @@ mod tests { } #[test] - fn seam_payload_is_inserted_before_the_last_body_close() { - let document = br#"

article

"#; + fn the_seam_payload_replaces_the_transform_owned_placeholder() { + let document = + format!(r#"

article

{TEMPLATE_SEAM_PLACEHOLDER}"#).into_bytes(); - let inserted = insert_before_body_close(document.to_vec(), b""); + let replaced = replace_seam_placeholder(document, b"") + .expect("should substitute the placeholder the transform emitted"); assert_eq!( - inserted, - br#"

article

"#, - "should ignore body-close text inside an earlier script and preserve tag casing" + replaced, br#"

article

"#, + "should splice the payload exactly where the parser marked the body end" ); } #[test] - fn seam_payload_is_appended_when_the_document_has_no_body_close() { - let inserted = - insert_before_body_close(b"
fragment
".to_vec(), b""); + fn a_body_close_written_in_script_data_does_not_attract_the_seam() { + // The reverse byte search this replaced picked the string literal, spliced a + // `` terminated the + // publisher's script — in the served page and in the stored template alike. + let document = format!( + r#"

article

{TEMPLATE_SEAM_PLACEHOLDER}"# + ) + .into_bytes(); + + let replaced = replace_seam_placeholder(document, b"") + .expect("should substitute the placeholder the transform emitted"); assert_eq!( - inserted, b"
fragment
", - "should append the seam payload to an HTML fragment" + replaced, + br#"

article

"#, + "should leave a body-close sequence inside script data untouched" + ); + } + + #[test] + fn a_publisher_copy_of_the_placeholder_refuses_substitution() { + let document = format!( + "

{TEMPLATE_SEAM_PLACEHOLDER}

article{TEMPLATE_SEAM_PLACEHOLDER}" + ) + .into_bytes(); + + let (returned, error) = + replace_seam_placeholder(document.clone(), b"") + .expect_err("should refuse a document that collides with the placeholder"); + + assert!( + matches!(error, SeamError::Repeated), + "should name the collision rather than guess an occurrence" + ); + assert_eq!( + returned, document, + "should hand back the publisher document byte for byte" + ); + } + + #[test] + fn a_document_without_the_placeholder_refuses_substitution() { + let document = b"

article

".to_vec(); + + let (returned, error) = + replace_seam_placeholder(document.clone(), b"") + .expect_err("should refuse a document the transform did not mark"); + + assert!( + matches!(error, SeamError::Missing), + "should name the absent placeholder rather than append blindly" + ); + assert_eq!( + returned, document, + "should hand back the document unchanged" ); } @@ -10129,6 +10286,176 @@ mod tests { ); } + #[tokio::test] + async fn a_meta_delivered_nonce_policy_bypasses_the_shared_template_cache() { + // The response-header gate cannot see this policy at all. Storing the document + // would replay one response's nonce to every later reader — the exact thing the + // header check exists to prevent. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for _ in 0..2 { + stub.push_response_with_headers( + 200, + br#"origin"# + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); + } + + let cold = run(&settings, &services, navigation_request()).await; + assert_eq!( + cold.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("bypass-response"), + "the cold response must refuse to store a nonce-bearing document" + ); + let _ = body_of(cold).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "the warm request must reach the origin, not a replayed nonce" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "nothing nonce-bearing may reach the shared cache" + ); + } + + #[tokio::test] + async fn a_nonce_attribute_without_a_policy_bypasses_the_shared_template_cache() { + // Fail closed: the attributes say the document was written for a per-response + // policy, whether or not the policy itself survived to this scan. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for _ in 0..2 { + stub.push_response_with_headers( + 200, + br#"origin"# + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); + } + + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + + assert_eq!(stub.recorded_request_uris().len(), 2); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty() + ); + } + + #[tokio::test] + async fn a_body_close_in_script_data_corrupts_neither_the_cold_nor_the_warm_document() { + // No structural `` anywhere: a reverse byte search takes the string + // literal, and the payload's `` then terminates the publisher's script + // — in the response served cold and in the template every warm reader gets. + const PUBLISHER_SCRIPT: &str = r#""#; + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + format!("{PUBLISHER_SCRIPT}

article

").into_bytes(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); + + let cold = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the cold document should be UTF-8"); + let warm = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the warm document should be UTF-8"); + + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the document is otherwise shareable and must still be stored and reused" + ); + for (label, document) in [("cold", &cold), ("warm", &warm)] { + assert!( + document.contains(PUBLISHER_SCRIPT), + "the {label} document must carry the publisher's script byte for byte: {document}" + ); + assert!( + !document.contains(AD_ASSEMBLY_SEAM), + "the {label} document must not ship an unresolved seam: {document}" + ); + } + } + + #[tokio::test] + async fn a_body_close_in_trailing_comment_data_does_not_attract_the_seam() { + // The reverse search took the *last* `` sequence, so the seam landed + // inside this comment and the assembled bids never executed. + const TRAILING_COMMENT: &str = ""; + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + format!("

article

{TRAILING_COMMENT}") + .into_bytes(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); + + let cold = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the cold document should be UTF-8"); + let warm = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the warm document should be UTF-8"); + + assert_eq!(stub.recorded_request_uris().len(), 1); + for (label, document) in [("cold", &cold), ("warm", &warm)] { + assert!( + document.contains(TRAILING_COMMENT), + "the {label} document must leave the publisher comment intact: {document}" + ); + assert!( + !document.contains(AD_ASSEMBLY_SEAM), + "the {label} document must not ship an unresolved seam: {document}" + ); + assert!( + !document.contains(TEMPLATE_SEAM_PLACEHOLDER), + "the transform-owned placeholder must never reach a reader: {document}" + ); + } + } + #[tokio::test] async fn a_post_is_never_answered_from_a_cached_get() { // `handle_publisher_request` is the `*`-method fallback route, so a publisher @@ -13403,6 +13730,7 @@ mod tests { let body = EdgeBody::from(compressed); let params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -13455,6 +13783,7 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -13496,6 +13825,7 @@ mod tests { let registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -13615,6 +13945,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -13672,6 +14003,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -13732,6 +14064,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -13792,6 +14125,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -13852,6 +14186,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -13900,6 +14235,7 @@ mod tests { fn non_html_stream_params(content_encoding: &str) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -14092,6 +14428,7 @@ mod tests { let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -14160,6 +14497,7 @@ mod tests { let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -14230,6 +14568,7 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -14293,6 +14632,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); let params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -14430,6 +14770,7 @@ mod tests { dispatched_auction: Option, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -14779,6 +15120,7 @@ mod tests { let ec_context = EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -14965,6 +15307,7 @@ mod tests { .map(bytes::Bytes::copy_from_slice) .collect(); let params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -15039,6 +15382,7 @@ mod tests { r#""#; let state = AdBidsState::with_script(bids_script); let params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -15096,6 +15440,7 @@ mod tests { // Claim gzip encoding but feed non-gzip bytes. The GzDecoder will // error as soon as it tries to read the gzip header. let params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -15208,6 +15553,7 @@ mod tests { let body = EdgeBody::from(html.to_vec()); let params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -15269,6 +15615,7 @@ mod tests { // Small, single-fragment RSC script — placeholder path (not fallback). let html = br#""#; let params = OwnedProcessResponseParams { + csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), From f73a4f5b70a3184c442a28ffcd372336a46ecf79 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 13:20:20 +0530 Subject: [PATCH 379/395] docs: design round 3 review remediation --- ...-1013-round-3-review-remediation-design.md | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-20-pr-1013-round-3-review-remediation-design.md diff --git a/docs/superpowers/specs/2026-08-20-pr-1013-round-3-review-remediation-design.md b/docs/superpowers/specs/2026-08-20-pr-1013-round-3-review-remediation-design.md new file mode 100644 index 000000000..c2ab9b52d --- /dev/null +++ b/docs/superpowers/specs/2026-08-20-pr-1013-round-3-review-remediation-design.md @@ -0,0 +1,65 @@ +# PR #1013 Round-3 Review Remediation Design + +## Goal + +Resolve every actionable round-2 and round-3 review finding on PR #1013 while preserving the existing template-cache architecture and avoiding unrelated refactoring. + +## Scope + +This remediation covers the page-bids terminal-privacy gap, the bootstrap scheduler test drift, the bootstrap-to-bundle one-shot handoff, explicit Fastly reservation cancellation, the remaining focused test and documentation gaps, and rerunning the browser integration job. + +The reviewer-recorded inert seam residue and `identity;q=0` behavior remain unchanged because they are explicitly accepted. Enforcing the terminal-private marker in adapters that do not apply late `RequestFilterEffects` remains out of scope. Workflow timeout or caching changes also remain out of scope unless a rerun reproduces the Playwright installation stall and the user separately chooses to address CI infrastructure. + +## Approach + +Use narrow, test-driven changes grouped by invariant. Prefer shared helpers for security behavior, but do not consolidate unrelated response construction or redesign the cache. + +### Terminal-private page-bids responses + +Introduce a generically named core helper that applies the existing `private, no-store` policy and inserts `TerminalPrivateResponse`. Keep the synthesized-HTML helper as the HTML-specific entry point, implemented in terms of the generic helper, so existing HTML call sites remain expressive. + +Route every page-bids response shape that Trusted Server marks private through the generic helper: preflight denial, unknown-format failure, successful JSON, and the synthesized invalid-304 response. This preserves the typed marker until Fastly's terminal hook applies late response mutations and then restores the privacy invariant. + +Tests will prove that page-bids response constructors attach the marker and that a Fastly `RequestFilterEffects` mutation attempting to set public CDN caching is removed before send. Coverage will include the successful per-user JSON path rather than relying only on a synthetic marker test. + +### GPT scheduler handoff and parity + +Store the initial-ad-init one-shot latch on the shared `tsjs` object rather than inside each installed scheduler closure. Both the bootstrap fallback and main bundle scheduler will consult and set the same internal boolean. Installing the bundle may replace the fallback function, but it cannot reset a scheduling decision already made for the document. + +The typed `TsjsApi` surface will document this field as internal lifecycle state. Tests will exercise the bootstrap-to-bundle handoff and prove the second scheduler cannot overwrite bids, slots, or schedule another `adInit`. The bootstrap suite will also mirror the bundle suite's omitted-versus-explicitly-empty `initialSlots` contract tests. + +### Fastly reservation release + +On body-length mismatch or metadata-encoding failure, `FastlyTemplateReservation::insert` will call `cancel_insert_or_update` before returning the validation error. If cancellation itself fails, return a `TemplateCacheError` that reports the cancellation failure with the original validation context attached or included, following the adapter's existing concrete error style. Successful validation retains the current insertion path. + +The direct-write comment will be corrected to state the actual safety mechanism: an unfinished write is never finished, and fallible reads plus declared-versus-observed body-length validation prevent partial data from being accepted. + +### Focused coverage and documentation cleanup + +The remediation will also: + +- add CR/LF rejection cases for metadata policy-header names and `content_type`, completing coverage of all encoded string fields; +- cover both comment-form and element-form publisher ESI in the end-to-end cache-bypass test; +- document that the `[nonce]` structural handler is the load-bearing CSP nonce safety net when a meta policy uses entity-encoded quotes; +- update the stale seam-collision assertion message to describe terminal emission and repeated-marker rejection rather than removed normalization; +- add a local C1/C3 glossary near the publisher cache terminology and use it to re-anchor the remaining references; +- qualify the configuration guide's request `max-age` inventory so `max-age=0` reload reuse agrees with the detailed paragraph; +- leave the explicitly accepted inert double-collision residue unchanged. + +## Error Handling + +Runtime error handling remains within the existing `TemplateCacheError` and `error-stack` boundaries. Terminal-private marking is infallible. Page-bids behavior and response bodies do not change; only their out-of-band privacy marker becomes complete. Cache validation failures still fail the store operation, but now deliberately discharge the Fastly transaction obligation before returning. + +## Verification + +Each behavioral change starts with a regression test and the narrowest relevant suite runs immediately after implementation. Final verification includes: + +- focused core publisher, response-privacy, HTML-processor, template-cache, and Fastly adapter tests; +- `cd crates/trusted-server-js/lib && npx vitest run`, the JS build, and JS formatting; +- `cargo fmt --all -- --check`; +- `cargo test-fastly`, `cargo test-axum`, `cargo test-cloudflare`, and `cargo test-spin`; +- `cargo clippy-fastly`, `cargo clippy-axum`, `cargo clippy-cloudflare`, `cargo clippy-cloudflare-wasm`, `cargo clippy-spin-native`, and `cargo clippy-spin-wasm`; +- documentation formatting; +- the parity and native CLI suites when their local prerequisites are available. + +After the implementation is pushed, rerun the failed browser integration job. A green rerun resolves the CI finding. A repeated timeout during Playwright installation will be recorded as runner-side infrastructure evidence and will not trigger an unapproved workflow redesign. From 6275d605486c28e5dc609acfa4be7172e5691342 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 13:23:45 +0530 Subject: [PATCH 380/395] docs: refine round 3 remediation design --- ...2026-08-20-pr-1013-round-3-review-remediation-design.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-08-20-pr-1013-round-3-review-remediation-design.md b/docs/superpowers/specs/2026-08-20-pr-1013-round-3-review-remediation-design.md index c2ab9b52d..0698bb74c 100644 --- a/docs/superpowers/specs/2026-08-20-pr-1013-round-3-review-remediation-design.md +++ b/docs/superpowers/specs/2026-08-20-pr-1013-round-3-review-remediation-design.md @@ -32,6 +32,8 @@ The typed `TsjsApi` surface will document this field as internal lifecycle state On body-length mismatch or metadata-encoding failure, `FastlyTemplateReservation::insert` will call `cancel_insert_or_update` before returning the validation error. If cancellation itself fails, return a `TemplateCacheError` that reports the cancellation failure with the original validation context attached or included, following the adapter's existing concrete error style. Successful validation retains the current insertion path. +Two Fastly Core Cache regression tests will reserve distinct cold keys, fail `insert` through body-length mismatch and metadata-encoding rejection respectively, assert the original validation reason, and immediately reserve the same key again. Receiving a second reservation proves the first transaction discharged its insert obligation instead of leaving collapsed waiters blocked. Because the simulator exposes no deterministic way to make `cancel_insert_or_update` itself fail, cancellation-error composition will be isolated behind a small private mapper and unit-tested with an injected failure value; that test will assert that both the original validation reason and cancellation failure remain visible. + The direct-write comment will be corrected to state the actual safety mechanism: an unfinished write is never finished, and fallible reads plus declared-versus-observed body-length validation prevent partial data from being accepted. ### Focused coverage and documentation cleanup @@ -41,7 +43,7 @@ The remediation will also: - add CR/LF rejection cases for metadata policy-header names and `content_type`, completing coverage of all encoded string fields; - cover both comment-form and element-form publisher ESI in the end-to-end cache-bypass test; - document that the `[nonce]` structural handler is the load-bearing CSP nonce safety net when a meta policy uses entity-encoded quotes; -- update the stale seam-collision assertion message to describe terminal emission and repeated-marker rejection rather than removed normalization; +- update both the stale seam-collision assertion message and the `queue_html_that_collides_with_the_marker` fixture documentation to describe terminal placeholder emission, repeated-marker rejection, and template-cache bypass rather than removed normalization; - add a local C1/C3 glossary near the publisher cache terminology and use it to re-anchor the remaining references; - qualify the configuration guide's request `max-age` inventory so `max-age=0` reload reuse agrees with the detailed paragraph; - leave the explicitly accepted inert double-collision residue unchanged. @@ -60,6 +62,7 @@ Each behavioral change starts with a regression test and the narrowest relevant - `cargo test-fastly`, `cargo test-axum`, `cargo test-cloudflare`, and `cargo test-spin`; - `cargo clippy-fastly`, `cargo clippy-axum`, `cargo clippy-cloudflare`, `cargo clippy-cloudflare-wasm`, `cargo clippy-spin-native`, and `cargo clippy-spin-wasm`; - documentation formatting; -- the parity and native CLI suites when their local prerequisites are available. +- `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity` as a required local gate, or explicit confirmation of the corresponding green `cargo test (cross-adapter parity)` CI check if an adapter executable prerequisite prevents a local run; +- the native CLI suite when its local prerequisites are available. After the implementation is pushed, rerun the failed browser integration job. A green rerun resolves the CI finding. A repeated timeout during Playwright installation will be recorded as runner-side infrastructure evidence and will not trigger an unapproved workflow redesign. From d8cc3e83770c3722fa0c63492bcf55a5eceb0c5e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 13:31:18 +0530 Subject: [PATCH 381/395] Plan round 3 review remediation --- ...8-20-pr-1013-round-3-review-remediation.md | 448 ++++++++++++++++++ 1 file changed, 448 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-20-pr-1013-round-3-review-remediation.md diff --git a/docs/superpowers/plans/2026-08-20-pr-1013-round-3-review-remediation.md b/docs/superpowers/plans/2026-08-20-pr-1013-round-3-review-remediation.md new file mode 100644 index 000000000..ad8f36e05 --- /dev/null +++ b/docs/superpowers/plans/2026-08-20-pr-1013-round-3-review-remediation.md @@ -0,0 +1,448 @@ +# PR #1013 Round-3 Review Remediation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve every actionable round-2 and round-3 PR #1013 review finding and return the branch with complete local verification and a rerun browser-integration check. + +**Architecture:** Preserve the existing publisher, Fastly terminal-hook, template-cache, and GPT scheduler boundaries. Complete the privacy invariant through a shared response marker helper, preserve GPT one-shot state on the shared `tsjs` object, explicitly discharge failed Fastly reservations, and make the remaining test/comment/doc changes locally without broad refactoring. + +**Tech Stack:** Rust 2024, `error-stack`, Fastly Core Cache/Viceroy, TypeScript, Vitest/jsdom, Cargo target aliases, GitHub Actions/CLI. + +--- + +## File Map + +- `crates/trusted-server-core/src/response_privacy.rs`: own generic terminal-private stamping and keep the synthesized-HTML wrapper. +- `crates/trusted-server-core/src/publisher.rs`: apply terminal-private marking to page-bids/invalid-304 responses; add page-bids and ESI coverage; repair cache terminology comments. +- `crates/trusted-server-adapter-fastly/src/main.rs`: prove late filter effects cannot weaken a page-bids response carrying the marker. +- `crates/trusted-server-adapter-fastly/src/template_cache.rs`: cancel invalid reservations, preserve error context, test released obligations, and correct partial-write documentation. +- `crates/trusted-server-core/src/platform/template_cache.rs`: complete CR/LF rejection coverage. +- `crates/trusted-server-core/src/html_processor.rs`: clarify CSP nonce safety net and collision assertion text. +- `crates/trusted-server-core/src/integrations/gpt_bootstrap.js`: use shared document-level latch state. +- `crates/trusted-server-js/lib/src/core/types.ts`: type and document the internal latch. +- `crates/trusted-server-js/lib/src/integrations/gpt/index.ts`: preserve the latch across fallback-to-bundle scheduler replacement. +- `crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts`: mirror slot semantics and cover bootstrap state. +- `crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts`: cover the real bootstrap-to-bundle handoff. +- `docs/guide/configuration.md`: qualify request-side `max-age` bypass wording. + +### Task 1: Complete terminal-private page-bids coverage + +**Files:** + +- Modify: `crates/trusted-server-core/src/response_privacy.rs:92` +- Modify: `crates/trusted-server-core/src/publisher.rs:4413,6014,6028,6365,17438` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs:580` + +- [ ] **Step 1: Write failing marker tests for page-bids response paths** + +In `publisher.rs`, add assertions using: + +```rust +assert!( + response + .extensions() + .get::() + .is_some(), + "page-bids response should remain terminal-private after late response effects" +); +``` + +Cover `page_bids_preflight_denied`, `page_bids_unknown_format`, and the successful JSON response returned by `run_page_bids_response`. Extend the invalid-origin-304 test to require the same marker. + +- [ ] **Step 2: Run the focused tests and verify failure** + +Run: + +```bash +cargo test-fastly page_bids -- --nocapture +cargo test-fastly eligible_navigation_rejects_unexpected_origin_304 -- --nocapture +``` + +Expected: the new marker assertions fail because these paths stamp only the header. + +- [ ] **Step 3: Add a generic terminal-private helper** + +In `response_privacy.rs`, add: + +```rust +pub(crate) fn enforce_terminal_private_cache_privacy(response: &mut Response) { + enforce_private_no_store(response); + response.extensions_mut().insert(TerminalPrivateResponse); +} +``` + +Change `enforce_synthesized_html_cache_privacy` to delegate to it. Keep both functions `pub(crate)` so no public API is added. + +- [ ] **Step 4: Route all affected response paths through the helper** + +Replace direct `Cache-Control: private, no-store` insertion for preflight denial, unknown format, successful page-bids JSON, and the invalid-origin-304 rebuild with `enforce_terminal_private_cache_privacy(&mut response)`. Preserve status, content type, body, and deprecated-alias headers. + +- [ ] **Step 5: Add the Fastly late-effects regression test** + +Construct a real page-bids denial response through `trusted_server_core::publisher::page_bids_preflight_denied`, apply `RequestFilterEffects` that sets public `Cache-Control` and CDN cache headers, then call `apply_terminal_response_effects`. Assert the terminal result is exactly `private, no-store`, has no validators/CDN cache headers, and retains the marker-driven behavior. Together with the core successful-JSON marker test, this pins the per-user JSON path without exporting test-only constructors. + +- [ ] **Step 6: Run focused and adapter tests** + +Run: + +```bash +cargo test-fastly page_bids -- --nocapture +cargo test-fastly late_filter_effects_cannot_make -- --nocapture +cargo test-fastly eligible_navigation_rejects_unexpected_origin_304 -- --nocapture +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: all pass. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/response_privacy.rs crates/trusted-server-core/src/publisher.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Keep page bids responses terminal private" +``` + +### Task 2: Preserve the GPT scheduler latch across handoff + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/gpt_bootstrap.js:82` +- Modify: `crates/trusted-server-js/lib/src/core/types.ts:430` +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts:648` +- Modify: `crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts:122` +- Modify: `crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts:150` + +- [ ] **Step 1: Mirror the missing bootstrap slot-contract tests** + +Add tests equivalent to the bundle suite: + +```typescript +it('fallback scheduler preserves head-injected slots when initialSlots is omitted', () => { + // Seed ts.adSlots, call scheduleInitialAdInit(bids), assert the same slots remain. +}) + +it('fallback scheduler replaces existing slots when initialSlots is explicitly empty', () => { + // Seed stale slots, call scheduleInitialAdInit({}, []), assert []. +}) +``` + +- [ ] **Step 2: Write a failing bootstrap-to-bundle handoff test** + +In `schedule_initial_ad_init.test.ts`, evaluate the verbatim bootstrap source, call its scheduler once, import the GPT module so it replaces the scheduler, then call the bundle scheduler with different bids/slots. Assert the first payload remains and only one load/double-rAF chain invokes `adInit`. + +- [ ] **Step 3: Run the focused Vitest files and verify the handoff test fails** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/integrations/gpt/gpt_bootstrap.test.ts test/integrations/gpt/schedule_initial_ad_init.test.ts +``` + +Expected: bootstrap slot tests pass against current behavior; handoff test fails because importing the bundle creates a fresh closure latch. + +- [ ] **Step 4: Type the shared internal state** + +Add to `TsjsApi` near `navGeneration`: + +```typescript +/** Internal one-shot state shared by bootstrap and bundle scheduler installs. */ +initialAdInitScheduled?: boolean; +``` + +Use the existing internal-field naming convention; do not expose a new callable API. + +- [ ] **Step 5: Replace both closure latches with shared state** + +In bootstrap JavaScript: + +```javascript +if ((ts.navGeneration || 0) !== 0 || ts.initialAdInitScheduled) return +ts.initialAdInitScheduled = true +``` + +In the TypeScript bundle: + +```typescript +if ((ts.navGeneration ?? 0) !== 0 || ts.initialAdInitScheduled) return +ts.initialAdInitScheduled = true +``` + +Update durable comments to say the state is one-shot per document and survives fallback-to-bundle scheduler replacement. + +- [ ] **Step 6: Run JS tests, build, and formatting verification** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/integrations/gpt/gpt_bootstrap.test.ts test/integrations/gpt/schedule_initial_ad_init.test.ts +npx vitest run +node build-all.mjs +npm run format +``` + +Expected: all pass, generated bundles build successfully, and Prettier reports every JS/TS file already formatted. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/gpt_bootstrap.js crates/trusted-server-js/lib/src/core/types.ts crates/trusted-server-js/lib/src/integrations/gpt/index.ts crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts +git commit -m "Preserve initial ad scheduler latch across handoff" +``` + +### Task 3: Explicitly cancel invalid Fastly reservations + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs:90,242,355` + +- [ ] **Step 1: Write failing obligation-release tests** + +For separate cold keys, obtain `TemplateCacheLookup::Reserved`, then: + +1. call `insert` with mismatched `body_len`; +2. call `insert` with metadata whose `content_type` contains `\n`. + +Assert each error contains the original validation reason. Immediately call `lookup_or_reserve` for the same key and require `Reserved`, proving the first transaction was canceled rather than left pending. + +- [ ] **Step 2: Write an error-composition unit test** + +Extract a private generic result mapper that accepts the validation error and a cancellation result. With an injected `Err("simulated cancellation failure")`, assert the returned `TemplateCacheError` text contains both the original validation reason and simulated cancellation failure. + +- [ ] **Step 3: Run Fastly cache tests and verify failure** + +Run: + +```bash +cargo test-fastly template_cache -- --nocapture +``` + +Expected: re-reservation tests fail or time out under the current implicit-drop behavior; the mapper test fails to compile until implemented. + +- [ ] **Step 4: Implement cancellation with preserved context** + +Create the validation error first, call `self.transaction.cancel_insert_or_update()`, and pass both values through the private mapper: + +```rust +fn invalid_reservation_result( + validation_error: TemplateCacheError, + cancellation: Result<(), E>, +) -> Result<(), TemplateCacheError> { + match cancellation { + Ok(()) => Err(validation_error), + Err(error) => Err(backend_error(format!( + "{validation_error}; cancelling invalid cache reservation also failed: {error:?}" + ))), + } +} +``` + +Use it on both pre-insert validation branches. Do not alter the transaction after `insert` consumes it. + +- [ ] **Step 5: Correct the direct-write partial-entry comment** + +State that `finish()` is deliberately skipped and readers reject partial content through fallible reads and the post-read check against declared `body_len`; do not claim the entry has no known length. + +- [ ] **Step 6: Run Fastly tests** + +Run: + +```bash +cargo test-fastly template_cache -- --nocapture +cargo test-fastly +``` + +Expected: all pass without blocked transaction lookups. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/template_cache.rs +git commit -m "Cancel invalid template cache reservations" +``` + +### Task 4: Close focused coverage and documentation gaps + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs:1102` +- Modify: `crates/trusted-server-core/src/html_processor.rs:725,2151` +- Modify: `crates/trusted-server-core/src/publisher.rs:1656,2078,5492,8225,8739,9246` +- Modify: `docs/guide/configuration.md:1430` + +- [ ] **Step 1: Add the missing metadata cases** + +Extend `metadata_encoding_rejects_line_break_injection` with a policy-header name containing `\r` and a `content_type` containing `\n`. Keep all four string-field cases in the same table-driven assertion. + +- [ ] **Step 2: Restore both ESI forms end to end** + +Parameterize `publisher_esi_comment_is_never_stored_or_executed` over: + +```rust +[ + "", + "publisher", +] +``` + +For each form, use a distinct cold cache/stub or distinct URL so each iteration independently proves bypass, byte preservation, no cache entry, and zero assembler calls. + +- [ ] **Step 3: Repair CSP and seam comments** + +Above the `[nonce]` handler, explain that meta `content` matching is supplemental because `lol_html` does not decode entity-encoded quotes; the structural `[nonce]` handler is the load-bearing refusal for any nonce an element can consume. Update the HTML processor assertion and publisher collision fixture rustdoc to describe terminal seam emission, repeated-marker rejection, and cache bypass. + +- [ ] **Step 4: Add the C1/C3 glossary and re-anchor references** + +Near the first surviving publisher cache reference, add a concise glossary: + +```rust +// C1 is Fastly's raw origin/read-through cache. C3 is the forbidden cache of a +// final per-user assembled response. The template cache sits between them. +``` + +Rewrite the remaining references so each is intelligible locally and does not mix the old C2 taxonomy with “template cache.” + +- [ ] **Step 5: Correct request max-age documentation** + +Change the fail-closed inventory to “positive or malformed request `max-age`, `min-fresh`” so it agrees with the `max-age=0` reload paragraph. + +- [ ] **Step 6: Run focused tests and formatting** + +Run: + +```bash +cargo test-fastly metadata_encoding_rejects_line_break_injection -- --nocapture +cargo test-fastly publisher_esi -- --nocapture +cargo test-fastly html_processor -- --nocapture +cargo fmt --all -- --check +cd docs +npx prettier --write guide/configuration.md +npm run format +``` + +Expected: all tests pass and formatters report no changes needed after formatting. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/platform/template_cache.rs crates/trusted-server-core/src/html_processor.rs crates/trusted-server-core/src/publisher.rs docs/guide/configuration.md +git commit -m "Close template cache review gaps" +``` + +### Task 5: Run the complete local CI gate + +**Files:** none unless a verification failure reveals an in-scope defect. + +- [ ] **Step 1: Verify the worktree diff and formatting** + +Run: + +```bash +git status --short +git diff --check main...HEAD +cargo fmt --all -- --check +cd crates/trusted-server-js/lib && npm run format +cd docs && npm run format +``` + +Expected: only planned changes exist; every formatter passes. + +- [ ] **Step 2: Run all target-matched tests** + +Run: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +./scripts/test-cli.sh +``` + +Expected: all pass. If only the CLI helper lacks a documented local prerequisite, record that explicitly; parity is required locally or must be confirmed green in CI. + +- [ ] **Step 3: Run all target-matched clippy gates** + +Run: + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: all pass with warnings denied. + +- [ ] **Step 4: Run final JS verification** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run +node build-all.mjs +``` + +Expected: all tests and builds pass. + +- [ ] **Step 5: Inspect the final diff** + +Run: + +```bash +git status --short --branch +git diff --stat origin/1009-esi-cacheable-root-spec...HEAD +git log --oneline origin/1009-esi-cacheable-root-spec..HEAD +``` + +Expected: the diff contains only the approved remediation and its spec/plan commits. + +### Task 6: Update the PR and rerun browser integration + +**Files:** none. + +- [ ] **Step 1: Push the reviewed commits** + +Run: + +```bash +git push origin 1009-esi-cacheable-root-spec +``` + +Expected: the PR head advances to the final local commit. + +- [ ] **Step 2: Locate the PR checks and browser workflow run** + +Run: + +```bash +gh pr view --json number,url,headRefOid,statusCheckRollup +gh pr checks +``` + +Identify the new-head `browser integration tests` check and its workflow run ID. Do not rerun an obsolete-head run. + +- [ ] **Step 3: Rerun only the failed browser job if needed** + +Before any rerun, mechanically verify the selected workflow run belongs to the current PR head: + +```bash +PR_HEAD_SHA=$(gh pr view --json headRefOid --jq .headRefOid) +RUN_HEAD_SHA=$(gh run view --json headSha --jq .headSha) +test "$RUN_HEAD_SHA" = "$PR_HEAD_SHA" +gh run rerun --job +``` + +Obtain `` from the selected run's jobs and only run the final command if the SHA comparison succeeds. This reruns the browser job alone rather than every failed job in the workflow. If the new push does not automatically schedule the browser job, locate the new-head workflow run rather than rerunning the old canceled run. Monitor until terminal state. + +Expected: green browser integration tests. If Playwright installation again consumes the job timeout before any browser launches, capture the run URL and exact failure phase as infrastructure evidence; do not change workflow caching or `timeout-minutes` without separate approval. + +- [ ] **Step 4: Report final verification and review mapping** + +Summarize each resolved finding, the local gate results, the browser-check result, commit hashes, and any infrastructure-only limitation. Do not claim completion until all required local gates and the relevant remote check have terminal evidence. From e70775802348cdafc94850449eeb9d3bd92a186a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 13:33:57 +0530 Subject: [PATCH 382/395] Ignore agent implementation worktrees --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 8ff935162..24b9e06aa 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,9 @@ src/*.html .specstory .vscode +# Agent implementation worktrees +/.worktrees/ + # Claude Code — ignore all, then whitelist shared config .claude/* !.claude/settings.json From 6ed133ac944915e2ecf3b422db3fd03124ec05be Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 13:42:35 +0530 Subject: [PATCH 383/395] Keep page bids responses terminal private --- .../trusted-server-adapter-fastly/src/main.rs | 25 ++++++++ crates/trusted-server-core/src/publisher.rs | 57 ++++++++++++++----- .../src/response_privacy.rs | 9 ++- 3 files changed, 74 insertions(+), 17 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 06f5f1aa7..2dd58ba88 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -607,6 +607,31 @@ mod tests { assert!(response.headers().get("etag").is_none()); } + #[test] + fn late_filter_effects_cannot_make_a_page_bids_response_public() { + let mut response = trusted_server_core::publisher::page_bids_preflight_denied(); + let effects = RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![ + HeaderMutation::set("cache-control", "public, s-maxage=3600"), + HeaderMutation::set("surrogate-control", "max-age=3600"), + HeaderMutation::set("cdn-cache-control", "public, max-age=3600"), + ], + }; + + apply_terminal_response_effects(&mut response, Some(&effects)); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("private, no-store") + ); + assert!(response.headers().get("surrogate-control").is_none()); + assert!(response.headers().get("cdn-cache-control").is_none()); + } + fn diagnostics_settings() -> Settings { Settings::from_toml( r#" diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index cc559975a..25e9cf078 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -65,7 +65,9 @@ use crate::platform::{ contains_publisher_esi_directive, }; use crate::price_bucket::{PriceGranularity, price_bucket}; -use crate::response_privacy::enforce_synthesized_html_cache_privacy; +use crate::response_privacy::{ + enforce_synthesized_html_cache_privacy, enforce_terminal_private_cache_privacy, +}; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{ @@ -4408,9 +4410,8 @@ pub async fn handle_publisher_request( .await; } - let response = Response::builder() + let mut response = Response::builder() .status(StatusCode::BAD_GATEWAY) - .header(header::CACHE_CONTROL, "private, no-store") .header(header::CONTENT_TYPE, "text/plain; charset=utf-8") .body(EdgeBody::from( "Publisher origin returned an invalid conditional response", @@ -4418,6 +4419,7 @@ pub async fn handle_publisher_request( .change_context(TrustedServerError::Proxy { message: "failed to build unexpected origin 304 response".to_string(), })?; + enforce_terminal_private_cache_privacy(&mut response); return Ok(PublisherResponse::Buffered(response)); } @@ -6014,10 +6016,7 @@ fn page_bids_request_allowed(req: &Request) -> bool { pub fn page_bids_preflight_denied() -> Response { let mut response = Response::new(EdgeBody::from("Forbidden")); *response.status_mut() = StatusCode::FORBIDDEN; - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); + enforce_terminal_private_cache_privacy(&mut response); response } @@ -6028,10 +6027,7 @@ pub fn page_bids_preflight_denied() -> Response { fn page_bids_unknown_format() -> Response { let mut response = Response::new(EdgeBody::from("Unknown format")); *response.status_mut() = StatusCode::BAD_REQUEST; - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); + enforce_terminal_private_cache_privacy(&mut response); response } @@ -6367,10 +6363,7 @@ pub async fn handle_page_bids( header::CONTENT_TYPE, HeaderValue::from_static("application/json"), ); - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); + enforce_terminal_private_cache_privacy(&mut response); mark_deprecated_alias(&mut response, is_legacy_alias); Ok(response) @@ -7305,6 +7298,26 @@ mod tests { .and_then(|v| v.to_str().ok()), Some("private, no-store") ); + assert!( + response + .extensions() + .get::() + .is_some(), + "page-bids errors should remain terminal-private after late response effects" + ); + } + + #[test] + fn a_preflight_denial_is_terminal_private() { + let response = page_bids_preflight_denied(); + + assert!( + response + .extensions() + .get::() + .is_some(), + "page-bids preflight denial should remain private after late response effects" + ); } } @@ -12090,6 +12103,13 @@ mod tests { Some("private, no-store"), "eligible origin 304 should return an explicitly non-storable response" ); + assert!( + response + .extensions() + .get::() + .is_some(), + "invalid origin 304 response should remain private after late response effects" + ); for header_name in [ header::ETAG, header::LAST_MODIFIED, @@ -17456,6 +17476,13 @@ mod tests { Some(&HeaderValue::from_static("application/json")), "should return JSON for `{path_and_format}`" ); + assert!( + response + .extensions() + .get::() + .is_some(), + "successful per-user page-bids JSON should remain terminal-private" + ); } } diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index af72b929b..81ae670f9 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -89,14 +89,19 @@ pub fn enforce_private_no_store(response: &mut Response) { strip_cdn_cache_headers(response); } +/// Marks a Trusted Server response as terminal-private and applies its cache policy. +pub(crate) fn enforce_terminal_private_cache_privacy(response: &mut Response) { + enforce_private_no_store(response); + response.extensions_mut().insert(TerminalPrivateResponse); +} + /// Forces synthesized HTML to be private and non-storable. /// /// Use this exact policy whenever Trusted Server changes an origin HTML /// representation with request-specific content: force `private, no-store`, /// remove origin validators, and remove all CDN-targeted cache directives. pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { - enforce_private_no_store(response); - response.extensions_mut().insert(TerminalPrivateResponse); + enforce_terminal_private_cache_privacy(response); } /// Forces cookie-bearing responses to stay private to shared caches. From 8abf9ae71e266aab057b2f672e361e726a5782dd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 13:44:39 +0530 Subject: [PATCH 384/395] Preserve initial ad scheduler latch across handoff --- .../src/integrations/gpt_bootstrap.js | 7 ++-- .../trusted-server-js/lib/src/core/types.ts | 2 + .../lib/src/integrations/gpt/index.ts | 15 +++---- .../integrations/gpt/gpt_bootstrap.test.ts | 35 ++++++++++++++++ .../gpt/schedule_initial_ad_init.test.ts | 41 +++++++++++++++++++ 5 files changed, 90 insertions(+), 10 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index cf0e61a3b..3e68f39c6 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -94,10 +94,11 @@ // and deliberately identical to the bundle scheduler — the impression is // spent on a viewed tab, and the post-hydration guarantee holds whenever // the request is actually issued. - var initialAdInitScheduled = false; ts.scheduleInitialAdInit = function (initialBids, initialSlots) { - if ((ts.navGeneration || 0) !== 0 || initialAdInitScheduled) return; - initialAdInitScheduled = true; + // The bundle may replace this scheduler after the fallback claims the initial + // pass. Keep the latch on the shared document API so replacement cannot reset it. + if ((ts.navGeneration || 0) !== 0 || ts.initialAdInitScheduled) return; + ts.initialAdInitScheduled = true; // Slots are generation-guarded for the same reason the bids are: the // shared-template seam sends both, and an assignment made before this call // would overwrite a committed SPA navigation's slots. diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 624b4ef92..3e21dd5be 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -429,6 +429,8 @@ export interface TsjsApi { gptSlotHandoffInternal?: boolean; /** Guards SPA pushState hook installation. */ spaHookInstalled?: boolean; + /** Internal one-shot state shared by bootstrap and bundle scheduler installs. */ + initialAdInitScheduled?: boolean; /** * Monotonic count of committed SPA navigations, incremented synchronously by * the SPA auction hook the moment it accepts a route change. The deferred diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index a58aa384d..0da5b4ad2 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -671,10 +671,12 @@ function installInitialLoadDetector(ts: TsjsApi): void { * their head script already installed the slots. An explicit empty array clears * that state, while omission preserves it. The scheduler accepts only its first * generation-0 call so duplicate public API calls cannot define and display the - * initial slots twice. If a navigation commits before scheduling or before the - * deferred callback, the SSR payload and `adInit()` are both dropped. The - * generation counter (not a URL comparison) keeps this aligned with the SPA - * auction hook's navigation identity. + * initial slots twice. The latch lives on `tsjs` so a bootstrap fallback that + * claims the initial pass keeps that claim when the bundle replaces its + * scheduler. If a navigation commits before scheduling or before the deferred + * callback, the SSR payload and `adInit()` are both dropped. The generation + * counter (not a URL comparison) keeps this aligned with the SPA auction hook's + * navigation identity. * * Hidden documents: browsers do not service `requestAnimationFrame` while a * document is hidden, so a background-tab load (Cmd+click, open-in-new-tab) @@ -686,13 +688,12 @@ function installInitialLoadDetector(ts: TsjsApi): void { * holds whenever the request is actually issued. */ function installScheduleInitialAdInit(ts: TsjsApi): void { - let initialAdInitScheduled = false; ts.scheduleInitialAdInit = function ( initialBids?: Record, initialSlots?: AuctionSlot[] ) { - if ((ts.navGeneration ?? 0) !== 0 || initialAdInitScheduled) return; - initialAdInitScheduled = true; + if ((ts.navGeneration ?? 0) !== 0 || ts.initialAdInitScheduled) return; + ts.initialAdInitScheduled = true; if (initialSlots !== undefined) ts.adSlots = initialSlots; if (initialBids !== undefined) ts.bids = initialBids; const runUnlessNavigated = (): void => { diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index 8ea28528d..c13c2f020 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -135,6 +135,41 @@ describe('gpt_bootstrap.js fallback', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('fallback scheduler preserves head-injected slots when initialSlots is omitted', () => { + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + const headSlot = { + id: 'head_slot', + gam_unit_path: '/123/head', + div_id: 'div-head', + formats: [[300, 250]] as Array<[number, number]>, + }; + ts.adSlots = [headSlot]; + + ts.scheduleInitialAdInit!({ head_slot: { hb_pb: '1.00' } }); + + expect(ts.adSlots).toEqual([headSlot]); + }); + + it('fallback scheduler replaces existing slots when initialSlots is explicitly empty', () => { + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + ts.adSlots = [ + { + id: 'stale_slot', + gam_unit_path: '/123/stale', + div_id: 'div-stale', + formats: [[300, 250]], + }, + ]; + + ts.scheduleInitialAdInit!({}, []); + + expect(ts.adSlots).toEqual([]); + }); + it('fallback scheduler rides animation frames in a hidden document, holding adInit until first view', () => { // Mirrors the bundle scheduler's intended hidden-tab behavior: rAF is not // serviced while hidden, so a background-tab load holds the initial diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts index 81a284596..727300aa1 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts @@ -1,3 +1,6 @@ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; + import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import type { TsjsApi } from '../../../src/core/types'; @@ -9,6 +12,14 @@ type TestWindow = Window & { const originalPushState = history.pushState.bind(history); const originalReplaceState = history.replaceState.bind(history); +const BOOTSTRAP_SOURCE = readFileSync( + path.resolve(process.cwd(), '../../trusted-server-core/src/integrations/gpt_bootstrap.js'), + 'utf8' +); + +function runBootstrap(): void { + new Function(BOOTSTRAP_SOURCE)(); +} /** * Executable lifecycle coverage for `tsjs.scheduleInitialAdInit` — the @@ -164,6 +175,36 @@ describe('scheduleInitialAdInit', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('keeps the first schedule claim across bootstrap-to-bundle handoff', async () => { + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + const firstSlot = { + id: 'first_slot', + gam_unit_path: '/123/first', + div_id: 'div-first', + formats: [[300, 250]] as Array<[number, number]>, + }; + const secondSlot = { + id: 'second_slot', + gam_unit_path: '/123/second', + div_id: 'div-second', + formats: [[728, 90]] as Array<[number, number]>, + }; + + ts.scheduleInitialAdInit!({ first_slot: { hb_pb: '1.00' } }, [firstSlot]); + await importGptModule(); + const adInit = vi.fn(); + ts.adInit = adInit; + ts.scheduleInitialAdInit!({ second_slot: { hb_pb: '2.00' } }, [secondSlot]); + + expect(ts.bids).toEqual({ first_slot: { hb_pb: '1.00' } }); + expect(ts.adSlots).toEqual([firstSlot]); + window.dispatchEvent(new Event('load')); + flushFrame(); + flushFrame(); + expect(adInit).toHaveBeenCalledTimes(1); + }); + it('still runs after a query-only history change before load', async () => { // The SPA auction hook identifies routes by pathname only, so a query-only // replaceState is not a navigation: it must neither trigger an auction nor From c8671cc931bc05e4d5909b81430c6ef499983a42 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 13:48:06 +0530 Subject: [PATCH 385/395] Cancel invalid template cache reservations --- .../src/template_cache.rs | 107 ++++++++++++++++-- 1 file changed, 100 insertions(+), 7 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs index fecbbb890..fe3148cb4 100644 --- a/crates/trusted-server-adapter-fastly/src/template_cache.rs +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -47,6 +47,18 @@ fn backend_error(message: impl Into) -> TemplateCacheError { } } +fn cancel_invalid_reservation( + validation_error: TemplateCacheError, + cancel: impl FnOnce() -> Result<(), E>, +) -> Result<(), TemplateCacheError> { + match cancel() { + Ok(()) => Err(validation_error), + Err(error) => Err(backend_error(format!( + "{validation_error}; cancelling invalid cache reservation also failed: {error:?}" + ))), + } +} + enum ReadFoundError { Invalid(TemplateCacheMiss), Backend(TemplateCacheError), @@ -103,15 +115,25 @@ impl PlatformTemplateCacheReservation for FastlyTemplateReservation { max_age: Duration, ) -> Result<(), TemplateCacheError> { if metadata.body_len != body.len() as u64 { - return Err(backend_error(format!( + let validation_error = backend_error(format!( "metadata body_len {} does not match the {} bytes supplied", metadata.body_len, body.len() - ))); + )); + return cancel_invalid_reservation(validation_error, || { + self.transaction.cancel_insert_or_update() + }); } - let encoded_metadata = metadata.encode().map_err(|error| { - backend_error(format!("encoding template metadata failed: {error}")) - })?; + let encoded_metadata = match metadata.encode() { + Ok(encoded_metadata) => encoded_metadata, + Err(error) => { + let validation_error = + backend_error(format!("encoding template metadata failed: {error}")); + return cancel_invalid_reservation(validation_error, || { + self.transaction.cancel_insert_or_update() + }); + } + }; let mut writer = self .transaction @@ -240,8 +262,9 @@ impl PlatformTemplateCache for FastlyTemplateCache { .map_err(|e| backend_error(format!("cache insert failed: {e:?}")))?; if let Err(e) = writer.write_all(&body) { - // Deliberately not calling `finish()`. An unfinished entry has no known - // length, and even if it is observable, `get`'s length check rejects it. + // Deliberately do not call `finish()`. If partial content becomes + // observable, fallible reads and the post-read check against the declared + // body length reject it. return Err(backend_error(format!("writing template body failed: {e}"))); } @@ -373,6 +396,76 @@ mod tests { } } + #[test] + fn length_mismatch_cancels_the_reservation_obligation() { + let cache = cache(); + let key = key("https://example.com/reservation-length-mismatch"); + let body = b"template".to_vec(); + let mut metadata = metadata_for(&body); + metadata.body_len += 1; + let reservation = match run(cache.lookup_or_reserve(&key)).expect("lookup should work") { + TemplateCacheLookup::Reserved(reservation) => reservation, + _ => panic!("a cold lookup should reserve the key"), + }; + + let error = reservation + .insert(&metadata, body, Duration::from_secs(60)) + .expect_err("length mismatch should fail insertion"); + + assert!( + error.to_string().contains("does not match"), + "should preserve the original validation reason: {error}" + ); + match run(cache.lookup_or_reserve(&key)).expect("second lookup should work") { + TemplateCacheLookup::Reserved(reservation) => reservation + .cancel() + .expect("should cancel the proof reservation"), + _ => panic!("the invalid insert should release the reservation obligation"), + } + } + + #[test] + fn metadata_encoding_failure_cancels_the_reservation_obligation() { + let cache = cache(); + let key = key("https://example.com/reservation-metadata-encoding"); + let body = b"template".to_vec(); + let mut metadata = metadata_for(&body); + metadata.content_type = "text/html\ninjected".to_string(); + let reservation = match run(cache.lookup_or_reserve(&key)).expect("lookup should work") { + TemplateCacheLookup::Reserved(reservation) => reservation, + _ => panic!("a cold lookup should reserve the key"), + }; + + let error = reservation + .insert(&metadata, body, Duration::from_secs(60)) + .expect_err("invalid metadata should fail insertion"); + + assert!( + error + .to_string() + .contains("encoding template metadata failed"), + "should preserve the original validation reason: {error}" + ); + match run(cache.lookup_or_reserve(&key)).expect("second lookup should work") { + TemplateCacheLookup::Reserved(reservation) => reservation + .cancel() + .expect("should cancel the proof reservation"), + _ => panic!("the invalid insert should release the reservation obligation"), + } + } + + #[test] + fn invalid_reservation_cancellation_preserves_both_errors() { + let error = cancel_invalid_reservation(backend_error("metadata validation failed"), || { + Err("simulated cancellation failure") + }) + .expect_err("invalid reservation should return an error"); + + let message = error.to_string(); + assert!(message.contains("metadata validation failed")); + assert!(message.contains("simulated cancellation failure")); + } + #[test] fn an_absent_key_is_a_miss_not_an_error() { let miss = From 733b66013ce13d8a6ca2b2291ef0da2247a9c20e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 13:51:18 +0530 Subject: [PATCH 386/395] Close template cache review gaps --- .../trusted-server-core/src/html_processor.rs | 7 +- .../src/platform/template_cache.rs | 17 +++ crates/trusted-server-core/src/publisher.rs | 125 ++++++++++-------- docs/guide/configuration.md | 2 +- 4 files changed, 89 insertions(+), 62 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 9f17f64bc..0a430258c 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -736,8 +736,9 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso } Ok(()) })); - // Nonce attributes are rejected on their own: a document carrying them is written - // for a per-response policy whether or not the policy itself reached this scan. + // `lol_html` does not entity-decode quoted meta CSP content for the check above. + // Reject nonce attributes independently so an entity-encoded meta policy cannot + // hide executable nonce-bound content from the template-cache safety scan. element_content_handlers.push(element!("[nonce]", move |_el| { observed.store(true, Ordering::SeqCst); Ok(()) @@ -2148,7 +2149,7 @@ mod tests { assert_eq!( html.matches(MARKER).count(), 2, - "one source occurrence plus one transform-owned template-cache seam must reach normalization" + "one source occurrence plus the transform-owned terminal seam must survive processing; repeated markers are rejected before template caching" ); assert!( html.ends_with(MARKER), diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index 76137201c..e3bbc42da 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -1118,6 +1118,23 @@ mod tests { schema_version: TEMPLATE_SCHEMA_VERSION, body_len: 0, }, + TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: vec![( + "content-security-policy\rh=link".to_string(), + "default-src 'self'".to_string(), + )], + content_type: "text/html".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 0, + }, + TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: Vec::new(), + content_type: "text/html\nh=link:".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 0, + }, ] { assert!( metadata.encode().is_err(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 25e9cf078..63f5d99bc 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1653,9 +1653,12 @@ pub async fn buffer_publisher_response_async( // `process_response_streaming_async`; inline transforms retain the origin // coding. This avoids recompressing and immediately decoding a full document. let bytes = output.into_inner(); + // Cache taxonomy for this path: C1 is the raw origin/read-through cache, + // the template cache stores processed reader-neutral HTML, and C3 would be + // a forbidden cache of the final per-user assembled response. // Store first, assemble second — never the reverse. The stored bytes are // shared between visitors; the assembled ones carry this visitor's bids. - // Swapping these two lines is the C3 leak. + // Swapping these two lines would create the forbidden C3 leak. // Read before the store: `store_template_if_authorized` *takes* the key so a // request cannot store twice, which would leave nothing for assembly to gate // on. @@ -2077,7 +2080,7 @@ impl core::error::Error for SeamError {} /// the publisher path stamps `private, no-store` and strips validators. Omitting it /// here does not fall back to a safe default — it emits HTML with no `Cache-Control` at /// all, which is heuristically cacheable by browsers and intermediaries. That is a -/// shared cache of an assembled per-user response: the C3 the design forbids outright. +/// forbidden C3 cache of a final per-user assembled response. /// /// Asserting the absence of `public`/`s-maxage`/`Surrogate-Control` would not have /// caught it. Nothing was present to forbid. @@ -5491,8 +5494,8 @@ impl TemplateCachePolicy { /// the most serious one that applies. /// /// See `docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md` -/// §6.6 for why C1, template cache and a final assembled-response cache are distinct, and why -/// the third must never exist. +/// §6.6 for why the C1 raw-origin/read-through cache, the reader-neutral template cache, and +/// the forbidden C3 final assembled-response cache are distinct. #[cfg(test)] pub(crate) fn template_cache_bypass_reason( mode: AssemblyMode, @@ -8235,59 +8238,63 @@ mod tests { } #[tokio::test] - async fn publisher_esi_comment_is_never_stored_or_executed() { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let assembler = Arc::new(RecordingTemplateAssembler::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services_with_assembler( - Arc::clone(&stub), - Arc::clone(&cache), - Arc::clone(&assembler), - ); - stub.push_response_with_headers( - 200, - b"".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ], - ); - - let response = run(&settings, &services, navigation_request()).await; - assert_eq!( - header_of( - &response, - header::HeaderName::from_static(HEADER_X_TS_TEMPLATE_CACHE), + async fn publisher_esi_directives_are_never_stored_or_executed() { + for (source, expected_directive) in [ + ("", ""), + ( + "publisher", + "publisher", ), - Some("bypass-response") - ); - assert_eq!( - header_of(&response, header::HeaderName::from_static("x-ts-assembly")), - Some("byte-seam-fallback") - ); - let document = String::from_utf8(body_of(response).await) - .expect("served document should be UTF-8"); + ] { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let assembler = Arc::new(RecordingTemplateAssembler::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services_with_assembler( + Arc::clone(&stub), + Arc::clone(&cache), + Arc::clone(&assembler), + ); + stub.push_response_with_headers( + 200, + format!("{source}").into_bytes(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); - assert!( - document - .to_ascii_lowercase() - .contains("") - ); - assert!(document.contains("scheduleInitialAdInit")); - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "publisher-authored ESI must never enter the shared template cache" - ); - assert_eq!( - assembler.calls.load(Ordering::Relaxed), - 0, - "publisher-authored ESI must never reach the platform parser" - ); + let response = run(&settings, &services, navigation_request()).await; + assert_eq!( + header_of( + &response, + header::HeaderName::from_static(HEADER_X_TS_TEMPLATE_CACHE), + ), + Some("bypass-response") + ); + assert_eq!( + header_of(&response, header::HeaderName::from_static("x-ts-assembly")), + Some("byte-seam-fallback") + ); + let document = String::from_utf8(body_of(response).await) + .expect("served document should be UTF-8"); + + assert!(document.to_ascii_lowercase().contains(expected_directive)); + assert!(document.contains("scheduleInitialAdInit")); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "publisher-authored ESI must never enter the shared template cache" + ); + assert_eq!( + assembler.calls.load(Ordering::Relaxed), + 0, + "publisher-authored ESI must never reach the platform parser" + ); + } } #[tokio::test] @@ -8748,8 +8755,9 @@ mod tests { /// Shareable HTML that already contains the seam marker. /// - /// The marker is reserved, but publisher content can still contain it. Fresh - /// normalization must disambiguate that content from the transform-owned seam. + /// The marker is reserved, but publisher content can still contain it. The + /// transform adds its own terminal placeholder; repeated markers then make the + /// response bypass the template cache rather than requiring normalization. fn queue_html_that_collides_with_the_marker(stub: &StubHttpClient) { stub.push_response_with_headers( 200, @@ -9256,7 +9264,8 @@ mod tests { #[tokio::test] async fn the_cached_template_holds_the_marker_and_never_the_bids() { - // The ordering the C3 prohibition depends on: store before assembling. If + // Store the reader-neutral template before assembling the final per-user + // response, which must never enter the forbidden C3 cache. If // these were swapped, the cache would hold one visitor's bids and serve them // to the next — and every test above would still pass, because the served // page would look correct. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index bec22b7cb..7a36afb28 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1427,7 +1427,7 @@ The cache fails closed. A template is stored only for a `GET` with a processable positive shared freshness. `private`, `no-store`, `no-cache`, exhausted or malformed freshness, `Set-Cookie`, `Vary: *`, `Vary: Cookie`, uncovered `Vary` names, response-bound CSP nonces, authorization, diagnostics sessions, range or -conditional requests, request-side `max-age`/`min-fresh` constraints, and +conditional requests, positive or malformed request `max-age`, `min-fresh`, and unsupported CDN-specific cache policy fields all bypass the template cache. Fastly `Surrogate-Control` is the narrow exception: the template cache accepts exactly one positive `max-age` plus optional valid `stale-while-revalidate` and `stale-if-error` From e2ef9f24817c703d20b81f7aed78590bfba4d035 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 13:59:05 +0530 Subject: [PATCH 387/395] Mark page bids response as must use --- crates/trusted-server-core/src/publisher.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 63f5d99bc..037f89705 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -6016,6 +6016,7 @@ fn page_bids_request_allowed(req: &Request) -> bool { /// trigger real PBS/APS auctions from a visitor's browser. Every adapter returns /// this same response for `OPTIONS /_ts/page-bids` and for its deprecated /// `/__ts/page-bids` alias. +#[must_use] pub fn page_bids_preflight_denied() -> Response { let mut response = Response::new(EdgeBody::from("Forbidden")); *response.status_mut() = StatusCode::FORBIDDEN; From b836fad6cc10418c8ae49ec992c54b04682f0c52 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 14:16:27 +0530 Subject: [PATCH 388/395] Resolve round-4 review on ad-template generation Blocking: - Refuse an unreadable `[creative_opportunities]` section instead of reading it as absent, which let a merge replace the operator's whole slot array. - Tell one re-rendered element apart from two colliding elements by comparing what the ephemeral markers did not cover, so a React SSR/hydration pair no longer refuses itself (a fully per-render publisher generated zero slots). - Refuse volatile div-id families by token shape rather than a hardcoded vendor name, covering every placement after the token instead of two. - Carry the ambiguous-stem verdict site-wide, so a landing page that renders one member of a refused group cannot resurrect the prefix. - Read only ISO 639-1 codes as a locale prefix, so `/tv`, `/ai` and `/us` stay section roots. - Track line endings past comments and single-line strings, so a stray triple quote no longer flips a CRLF config to LF. - Report evidence truncation instead of dropping entries silently, and align the Rust cap with the collector's. - Escape config-derived slot ids in `ts config ad-templates check` output. Non-blocking: - Adopt an inferred section policy when the config has none: a `{section}` slot without `section_root` cannot load, so there is no policy to preserve. - Note a followed root redirect; keep credentials, queries, and origins out of per-page notes and the cross-origin refusal. - Report per-page collection failures once and the consent stub once per run. - Collapse index-document links onto their section. - Expose the browser flags on `ts audit generate` and its legacy alias. - Move the dry-run "no changes" sentence to stderr and build the diff lazily. - Pace the crawl before announcing the page; scope audit cookies by origin. - Make the consent stub configurable and enumerable so a CMP that installs via `defineProperty` is not aborted, and the stub is not a fingerprint. Docs and debt: correct the strict-mode claim for sizeless out-of-page slots, document both new refusal classes and the stderr progress contract, drop the real publisher and vendor identifiers from the spec, order the manifest dependencies, and document the arms and fields that are unreachable or reserved. --- crates/trusted-server-cli/Cargo.toml | 4 +- .../src/ad_templates/compare.rs | 16 +- .../src/ad_templates/output.rs | 4 + crates/trusted-server-cli/src/app_config.rs | 2 +- .../commands/audit/ad_template_collector.js | 25 +- .../src/commands/audit/ad_templates.rs | 11 +- .../src/commands/audit/browser.rs | 58 ++- .../src/commands/audit/collector.rs | 23 + .../src/commands/audit/consent_stub.js | 9 +- .../audit/generate/browser_collector.rs | 135 +++--- .../src/commands/audit/generate/collector.rs | 22 +- .../src/commands/audit/generate/crawl_plan.rs | 190 +++++++- .../src/commands/audit/generate/evidence.rs | 88 +++- .../src/commands/audit/generate/gpt_slots.rs | 436 +++++++++++++----- .../src/commands/audit/generate/mod.rs | 136 +++++- .../src/commands/audit/generate/slot_toml.rs | 134 ++++-- .../commands/audit/generate/unit_template.rs | 9 +- .../src/commands/audit/mod.rs | 96 +++- .../src/commands/audit/page.rs | 81 +++- .../src/commands/config/ad_templates.rs | 12 +- docs/guide/cli.md | 51 +- .../2026-06-26-server-side-ad-template-cli.md | 9 +- .../2026-08-18-pr-823-review-resolution.md | 3 +- ...26-08-19-refuse-volatile-div-collisions.md | 8 +- ...6-08-18-pr-823-review-resolution-design.md | 8 +- ...9-refuse-volatile-div-collisions-design.md | 70 ++- 26 files changed, 1276 insertions(+), 364 deletions(-) diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index 20c454a70..e08114850 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -17,8 +17,8 @@ workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] chromiumoxide = { workspace = true } clap = { workspace = true } -edgezero-core = { workspace = true } edgezero-cli = { workspace = true } +edgezero-core = { workspace = true } futures = { workspace = true } glob = { workspace = true } http = { workspace = true } @@ -30,9 +30,9 @@ serde_json = { workspace = true } similar = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } -tracing = { workspace = true } toml = { workspace = true } toml_edit = { workspace = true } +tracing = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } which = { workspace = true } diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs index a1dc271a1..48ab71f3e 100644 --- a/crates/trusted-server-cli/src/ad_templates/compare.rs +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -45,11 +45,16 @@ pub struct GptSlotEvidence { pub phase: EvidencePhase, } -/// An `apstag.fetchBids` call observed on the page (spec §5.5). +/// An `apstag.fetchBids` call the page made, if any were recorded. +/// +/// The collector no longer hooks `apstag`: server-side APS configuration is +/// metadata rather than a client assertion, so a missing client call is not a +/// finding. The field and this shape stay for the evidence payload's schema, and +/// the list arrives empty. #[derive(Debug, Clone, Deserialize)] #[allow( dead_code, - reason = "decoded for compatibility; APS slot IDs are server-side metadata, not a client assertion" + reason = "decoded for schema stability; the collector records no APS calls" )] pub struct ApsFetchBidsEvidence { /// The APS slot ID requested. @@ -188,7 +193,8 @@ pub struct SlotEvidence { /// Live ad-slot evidence with no matching configured slot. #[derive(Debug, Clone)] pub struct ExtraEvidence { - /// Evidence kind: `dom`, `gpt`, or `aps`. + /// Evidence kind. Only `gpt` is produced today; the field is a string so a + /// later evidence source can be added without changing the JSON schema. pub kind: String, /// The phase it was observed in. pub phase: EvidencePhase, @@ -267,6 +273,10 @@ pub fn compare_page_evidence( let banner = banner_sizes(slot); let mut warnings = Vec::new(); + // `expected_slots_for_path` drops a slot whose template does not render, + // so on the verify path this arm is unreachable; it exists for callers + // that build expected slots directly, and as a guard if that filter ever + // changes. if slot.gam_unit_path.is_none() { warnings.push(warning( "gam_unit_path_unrenderable", diff --git a/crates/trusted-server-cli/src/ad_templates/output.rs b/crates/trusted-server-cli/src/ad_templates/output.rs index 6321b1707..e12c9eebc 100644 --- a/crates/trusted-server-cli/src/ad_templates/output.rs +++ b/crates/trusted-server-cli/src/ad_templates/output.rs @@ -129,6 +129,10 @@ pub struct VerificationReport { /// One entry per requested URL, in input order. pub pages: Vec, /// Run-level warnings not attributable to a single page. + /// + /// Always empty today — every warning the verifier raises belongs to a page + /// or a slot. Kept because the JSON schema declares it, so a consumer can + /// read it unconditionally. pub warnings: Vec, } diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs index a54cdfc72..bee536146 100644 --- a/crates/trusted-server-cli/src/app_config.rs +++ b/crates/trusted-server-cli/src/app_config.rs @@ -63,7 +63,7 @@ pub fn load_settings(args: &AppConfigArgs) -> Result { /// Returns the same path-resolution, read, and parse errors as /// [`load_settings`]. #[cfg(test)] -pub fn load_file_settings(args: &AppConfigArgs) -> Result { +pub(crate) fn load_file_settings(args: &AppConfigArgs) -> Result { load_settings_with_env_overlay(args, false) } diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js index 46d1485d4..6938808f5 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -1,7 +1,7 @@ // Bounded ad-template evidence collector, injected before publisher scripts run. // -// This body runs inside an IIFE that defines `__TS_CONFIG` (configured div -// prefixes + APS slot IDs). It records evidence into `window.__tsAdTemplateEvidence` +// This body runs inside an IIFE that defines `__TS_CONFIG` (the configured div +// prefixes). It records evidence into `window.__tsAdTemplateEvidence` // and never captures page HTML, cookies, storage, request bodies, or arbitrary DOM. // It always calls original page functions with unchanged arguments and never // spoofs the browser automation flag. @@ -28,8 +28,22 @@ function __ts_text(value) { return String(value).slice(0, __ts_max_string_length) } +// Truncation has to be visible: surplus configured slots classify Missing, and +// `--strict` counts that, so a silent drop is indistinguishable from real drift. +let __ts_truncated = false function __ts_push(list, entry) { - if (list.length < __ts_max_entries) list.push(entry) + if (list.length < __ts_max_entries) { + list.push(entry) + return + } + if (__ts_truncated) return + __ts_truncated = true + if (__ts_ev.warnings.length < __ts_max_entries) { + __ts_ev.warnings.push({ + code: "evidence_truncated", + message: "an evidence list hit the " + __ts_max_entries + "-entry cap; results are incomplete" + }) + } } function __ts_warn(code, error) { @@ -52,7 +66,7 @@ function __ts_warn_ignored_size(width, height) { numeric && (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) __ts_push(__ts_ev.warnings, { code: outOfRange ? "size_out_of_range" : "fluid_size_ignored", - message: outOfRange ? "GPT size outside u32 range ignored" : "non-numeric GPT size ignored" + message: outOfRange ? "GPT size outside u32 range ignored" : "non-integer GPT size ignored" }) } @@ -131,6 +145,9 @@ function __ts_install(name, wrap) { let internal Object.defineProperty(window, name, { configurable: true, + // A real `window.googletag` is an ordinary enumerable global; matching that + // keeps `Object.keys(window)` identical with and without the collector. + enumerable: true, get() { return internal }, diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs index 72c1733e1..cb11a0a0c 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -164,6 +164,13 @@ fn build_report( }) } +/// The URL without its fragment, for comparisons the server can observe. +pub(super) fn without_fragment(url: &url::Url) -> url::Url { + let mut url = url.clone(); + url.set_fragment(None); + url +} + /// Whether navigation left the requested URL's origin (scheme, host, or port). /// /// A same-host default-port `http:80` to `https:443` redirect is *not* a change: @@ -245,7 +252,9 @@ fn build_page( code: format!("page_{}", warning.code), message: warning.message.clone(), })); - if requested != final_url { + // Fragments never reach the server, so a fragment-only difference is not a + // redirect and slots match on the path either way. + if without_fragment(requested) != without_fragment(final_url) { warnings.push(Warning { code: "redirected".to_string(), message: format!("navigation redirected from {requested} to {final_url}"), diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs index 102443dce..b6bbdacd5 100644 --- a/crates/trusted-server-cli/src/commands/audit/browser.rs +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -38,9 +38,14 @@ const SETTLE_POLL_MS: u64 = 250; const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); /// Bound for each CDP operation after navigation. const CDP_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); -/// Hard cap per decoded evidence list, mirroring the collector script's -/// `__ts_max_entries`, so a hostile page cannot inflate CLI memory. -const MAX_EVIDENCE_ENTRIES: usize = 1024; +/// Hard cap per decoded evidence list, so a hostile page cannot inflate CLI +/// memory. +/// +/// Must equal `__ts_max_entries` in `ad_template_collector.js`. The collector +/// already caps each list, but the evidence object lives on `window`, so a page +/// that appends to it directly is bounded here instead. Anything the collector +/// itself dropped is reported as an `evidence_truncated` warning. +const MAX_EVIDENCE_ENTRIES: usize = 128; /// Hard cap on the UTF-8 JSON payload before CDP transfers it back to Rust. const MAX_EVIDENCE_PAYLOAD_BYTES: usize = 1024 * 1024; /// Hard cap on browser teardown so a wedged Chrome cannot hang the audit. @@ -239,11 +244,19 @@ pub(crate) fn resolve_chrome( } /// Builds a host-only cookie that applies to every path on `url`'s host. +/// +/// Scoped by origin rather than by the full URL: only the origin is load-bearing +/// for a host-only cookie, and a full URL would carry the path, query, and any +/// `user:password@` into CDP and into this function's error message. pub(crate) fn host_cookie(name: &str, value: &str, url: &url::Url) -> Result { - url.host_str() - .ok_or_else(|| format!("cannot scope cookie `{name}` because {} has no host", url))?; + let origin = url.origin(); + if !origin.is_tuple() { + return Err(format!( + "cannot scope cookie `{name}` because the audited URL has no host" + )); + } let mut cookie = CookieParam::new(name.to_string(), value.to_string()); - cookie.url = Some(url.to_string()); + cookie.url = Some(origin.ascii_serialization()); cookie.path = Some("/".to_string()); cookie.secure = Some(url.scheme() == "https"); Ok(cookie) @@ -889,6 +902,23 @@ fn decode_ad_evidence_envelope( } } +/// Whether a Chrome/Chromium fixture is available for browser-backed tests. +/// +/// Skips optional local runs, but makes the scripted/CI contract fail loudly. +/// Shared with the generation collector's tests so the contract has one +/// definition. +#[cfg(test)] +pub(crate) fn browser_fixture_available() -> bool { + if resolve_chrome(None).is_ok() { + return true; + } + assert!( + std::env::var_os("TS_AUDIT_BROWSER_TESTS").is_none(), + "TS_AUDIT_BROWSER_TESTS requires Chrome/Chromium; set CHROME to its executable" + ); + false +} + #[cfg(test)] mod tests { use std::io::{Read as _, Write as _}; @@ -900,18 +930,6 @@ mod tests { AdTemplateCollectorConfig, build_ad_template_init_script, }; - /// Skips optional local runs, but makes the scripted/CI contract fail loudly. - fn browser_fixture_available() -> bool { - if resolve_chrome(None).is_ok() { - return true; - } - assert!( - std::env::var_os("TS_AUDIT_BROWSER_TESTS").is_none(), - "TS_AUDIT_BROWSER_TESTS requires Chrome/Chromium; set CHROME to its executable" - ); - false - } - #[test] fn well_known_chrome_paths_are_known_for_this_os() { // macOS/Linux/Windows each have candidate paths; guards the cfg branches. @@ -941,8 +959,8 @@ mod tests { assert_eq!(cookie.path.as_deref(), Some("/")); assert_eq!( cookie.url.as_deref(), - Some("https://publisher.example/news/story"), - "the URL scopes a host-only cookie before first navigation" + Some("https://publisher.example"), + "the origin scopes a host-only cookie before first navigation" ); assert_eq!(cookie.secure, Some(true), "HTTPS cookies must be Secure"); } diff --git a/crates/trusted-server-cli/src/commands/audit/collector.rs b/crates/trusted-server-cli/src/commands/audit/collector.rs index d803682ab..6ab427b2c 100644 --- a/crates/trusted-server-cli/src/commands/audit/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/collector.rs @@ -74,10 +74,33 @@ pub struct GenerateBrowserOpts { #[arg(long, default_value_t = 10_000)] pub settle_max_ms: u64, /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as the evidence it writes config from, so an + /// invalid certificate could mean an impersonator is harvesting the session + /// and fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. #[arg(long)] pub danger_accept_invalid_certs: bool, } +/// Defaults mirroring the `#[arg(default_value_t)]` values above, so a path that +/// builds these options in code (the legacy `ts audit ` form) behaves like +/// the parsed command. +impl Default for GenerateBrowserOpts { + fn default() -> Self { + Self { + chrome: None, + headful: false, + no_assume_consent: false, + browser_proxy: None, + settle_quiet_ms: 750, + settle_max_ms: 10_000, + danger_accept_invalid_certs: false, + } + } +} + impl GenerateBrowserOpts { /// Validates relationships between independently parsed browser flags. pub fn validate(&self) -> Result<(), String> { diff --git a/crates/trusted-server-cli/src/commands/audit/consent_stub.js b/crates/trusted-server-cli/src/commands/audit/consent_stub.js index 8a35da29e..27699cd1e 100644 --- a/crates/trusted-server-cli/src/commands/audit/consent_stub.js +++ b/crates/trusted-server-cli/src/commands/audit/consent_stub.js @@ -62,7 +62,14 @@ // keeps the deterministic audit answer without throwing and aborting // the publisher's CMP initialization. set: () => {}, - configurable: false + // Configurable so a CMP that installs itself with `defineProperty` + // replaces the stub instead of throwing: losing the substitution on + // such a page is better than aborting the CMP mid-initialization and + // auditing a half-built ad stack. Enumerable so the property looks like + // the real global it stands in for, rather than adding a signal that + // `Object.keys(window)` can see the difference. + configurable: true, + enumerable: true }) } catch (error) { // The page installed an earlier value; leave it untouched. diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index 99ba47728..469c20e68 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -17,8 +17,9 @@ use crate::commands::audit::browser::{ }; use crate::commands::audit::collector::GenerateBrowserOpts; use crate::commands::audit::generate::collector::{ - AuditCollector, CollectedGptSlot, CollectedLink, CollectedPage, CollectedRequest, - CollectedScriptTag, CollectionProgress, ControlFlow, PageSink, ProgressSink, RootPlanner, + AuditCollector, CONSENT_STUB_WARNING, CollectedGptSlot, CollectedLink, CollectedPage, + CollectedRequest, CollectedScriptTag, CollectionProgress, ControlFlow, PageSink, ProgressSink, + RootPlanner, }; use crate::error::{CliResult, report_error}; @@ -33,7 +34,10 @@ const SETTLE_MAX_WAIT: Duration = Duration::from_secs(12); const NAVIGATION_LOAD_TIMEOUT: Duration = Duration::from_secs(12); const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); const PAGE_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); -const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 100_000; +/// Size the page's resource-timing buffer is raised to before navigation, and +/// therefore also the count at which the buffer is full and entries were lost. +/// One constant so the script and the warning threshold cannot drift apart. +const RESOURCE_TIMING_BUFFER_SIZE: usize = 100_000; const RESOURCE_TIMING_BUFFER_WARNING: &str = "browser resource timing buffer reached its configured size; some network assets may be missing"; /// A device the crawl can emulate. @@ -397,6 +401,13 @@ async fn with_browser( } else { Some(targets.len()) }; + // Pace the crawl before announcing the page, so the progress line marks + // the navigation rather than the start of the wait. Back-to-back + // navigations are both discourteous to the origin and a signal bot + // protection scores against the session. + if index > 0 && !page_delay.is_zero() { + sleep(page_delay).await; + } if let Err(error) = on_progress(CollectionProgress::Loading { current: index + 1, total, @@ -405,11 +416,6 @@ async fn with_browser( result = Err(error); break; } - // Pace the crawl. Back-to-back navigations are both discourteous to the - // origin and a signal bot protection scores against the session. - if index > 0 && !page_delay.is_zero() { - sleep(page_delay).await; - } let collected = collect_page_from_browser( &mut browser, &target, @@ -502,13 +508,15 @@ async fn collect_page_from_browser( settle_quiet: Duration, settle_max: Duration, ) -> CliResult { - set_browser_cookies(browser, cookies, target_url) - .await - .map_err(report_error)?; + // Per-page failures below return the message unlogged: the crawl attributes + // each one to its page once, and `report_error` would also log an unscoped + // duplicate in the middle of progress output. + set_browser_cookies(browser, cookies, target_url).await?; - let page = browser.new_page("about:blank").await.map_err(|error| { - report_error(format!("failed to create browser page for audit: {error}")) - })?; + let page = browser + .new_page("about:blank") + .await + .map_err(|error| format!("failed to create browser page for audit: {error}"))?; let result = collect_open_page( &page, @@ -556,21 +564,14 @@ async fn collect_open_page( if assume_consent { page.evaluate_on_new_document(SHARED_CONSENT_STUB_SCRIPT) .await - .map_err(|error| { - report_error(format!("failed to install the consent stub: {error}")) - })?; - warnings.push( - "consent_stub_active: audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution" - .to_string(), - ); + .map_err(|error| format!("failed to install the consent stub: {error}"))?; + warnings.push(CONSENT_STUB_WARNING.to_string()); } - page.evaluate_on_new_document("performance.setResourceTimingBufferSize(100000)") - .await - .map_err(|error| { - report_error(format!( - "failed to increase the resource timing buffer: {error}" - )) - })?; + page.evaluate_on_new_document(format!( + "performance.setResourceTimingBufferSize({RESOURCE_TIMING_BUFFER_SIZE})" + )) + .await + .map_err(|error| format!("failed to increase the resource timing buffer: {error}"))?; // Navigate, but don't hard-fail when the `load` event never fires. Ad-heavy // pages (video players, continuous ad refresh, anti-bot scripts) can keep @@ -624,17 +625,17 @@ async fn collect_open_page( let final_url = timeout(PAGE_OPERATION_TIMEOUT, page.url()) .await - .map_err(|_| report_error("timed out reading final page URL"))? - .map_err(|error| report_error(format!("failed to read final page URL: {error}")))? - .ok_or_else(|| report_error("browser page URL was empty after navigation"))?; + .map_err(|_| "timed out reading final page URL".to_string())? + .map_err(|error| format!("failed to read final page URL: {error}"))? + .ok_or("browser page URL was empty after navigation")?; let page_title = timeout(PAGE_OPERATION_TIMEOUT, page.get_title()) .await - .map_err(|_| report_error("timed out reading page title"))? - .map_err(|error| report_error(format!("failed to read page title: {error}")))?; + .map_err(|_| "timed out reading page title".to_string())? + .map_err(|error| format!("failed to read page title: {error}"))?; let html = timeout(PAGE_OPERATION_TIMEOUT, page.content()) .await - .map_err(|_| report_error("timed out reading rendered page HTML"))? - .map_err(|error| report_error(format!("failed to read rendered page HTML: {error}")))?; + .map_err(|_| "timed out reading rendered page HTML".to_string())? + .map_err(|error| format!("failed to read rendered page HTML: {error}"))?; let script_tags: Vec = timeout( PAGE_OPERATION_TIMEOUT, @@ -646,14 +647,10 @@ async fn collect_open_page( ), ) .await - .map_err(|_| report_error("timed out reading rendered script tags"))? - .map_err(|error| report_error(format!("failed to read rendered script tags: {error}")))? + .map_err(|_| "timed out reading rendered script tags".to_string())? + .map_err(|error| format!("failed to read rendered script tags: {error}"))? .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode rendered script tag data: {error}" - )) - })?; + .map_err(|error| format!("failed to decode rendered script tag data: {error}"))?; let network_requests: Vec = timeout( PAGE_OPERATION_TIMEOUT, @@ -665,18 +662,10 @@ async fn collect_open_page( ), ) .await - .map_err(|_| report_error("timed out reading browser performance entries"))? - .map_err(|error| { - report_error(format!( - "failed to read browser performance resource entries: {error}" - )) - })? + .map_err(|_| "timed out reading browser performance entries".to_string())? + .map_err(|error| format!("failed to read browser performance resource entries: {error}"))? .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode browser performance resource data: {error}" - )) - })?; + .map_err(|error| format!("failed to decode browser performance resource data: {error}"))?; if let Some(warning) = resource_timing_buffer_warning(network_requests.len()) { warnings.push(warning.to_string()); @@ -753,9 +742,7 @@ async fn collect_open_page( .await_promise(true) .return_by_value(true) .build() - .map_err(|error| { - report_error(format!("failed to build sitemap evaluation: {error}")) - })?; + .map_err(|error| format!("failed to build sitemap evaluation: {error}"))?; match timeout(PAGE_OPERATION_TIMEOUT, page.evaluate(evaluation)).await { Ok(Ok(result)) => match result.into_value() { Ok(locations) => locations, @@ -975,23 +962,19 @@ async fn wait_for_page_settle( let ready_state: String = timeout(PAGE_OPERATION_TIMEOUT, page.evaluate("document.readyState")) .await - .map_err(|_| report_error("timed out reading document ready state"))? - .map_err(|error| { - report_error(format!("failed to read document ready state: {error}")) - })? + .map_err(|_| "timed out reading document ready state".to_string())? + .map_err(|error| format!("failed to read document ready state: {error}"))? .into_value() - .map_err(|error| { - report_error(format!("failed to decode document ready state: {error}")) - })?; + .map_err(|error| format!("failed to decode document ready state: {error}"))?; let resource_count: usize = timeout( PAGE_OPERATION_TIMEOUT, page.evaluate("performance.getEntriesByType('resource').length"), ) .await - .map_err(|_| report_error("timed out reading resource count"))? - .map_err(|error| report_error(format!("failed to read resource count: {error}")))? + .map_err(|_| "timed out reading resource count".to_string())? + .map_err(|error| format!("failed to read resource count: {error}"))? .into_value() - .map_err(|error| report_error(format!("failed to decode resource count: {error}")))?; + .map_err(|error| format!("failed to decode resource count: {error}"))?; // Accept `interactive` as well as `complete`: ad-heavy pages often never // reach `complete` (the `load` event never fires), but their GPT slots @@ -1056,8 +1039,7 @@ fn is_successful_navigation_status(status: i64) -> bool { } fn resource_timing_buffer_warning(resource_count: usize) -> Option<&'static str> { - (resource_count >= RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD) - .then_some(RESOURCE_TIMING_BUFFER_WARNING) + (resource_count >= RESOURCE_TIMING_BUFFER_SIZE).then_some(RESOURCE_TIMING_BUFFER_WARNING) } #[derive(Debug, Deserialize)] @@ -1081,18 +1063,7 @@ mod tests { use chromiumoxide::handler::http::HttpRequest; use super::*; - - /// Skips optional local runs, but makes the scripted/CI contract fail loudly. - fn browser_fixture_available() -> bool { - if resolve_chrome(None).is_ok() { - return true; - } - assert!( - std::env::var_os("TS_AUDIT_BROWSER_TESTS").is_none(), - "TS_AUDIT_BROWSER_TESTS requires Chrome/Chromium; set CHROME to its executable" - ); - false - } + use crate::commands::audit::browser::browser_fixture_available; #[test] fn successful_navigation_status_allows_redirects_but_rejects_errors() { @@ -1136,12 +1107,12 @@ mod tests { #[test] fn resource_timing_buffer_warning_starts_at_threshold() { assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD - 1), + resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_SIZE - 1), None, "should not warn before the resource timing buffer threshold" ); assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD), + resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_SIZE), Some(RESOURCE_TIMING_BUFFER_WARNING), "should warn when the resource timing buffer reaches the threshold" ); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs index 9e760a500..dc23af09c 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs @@ -3,6 +3,12 @@ use url::Url; use crate::error::CliResult; +/// Warning recorded on a page collected with the audit consent stub installed. +/// +/// A whole-run fact rather than a property of one page, so consumers report it +/// once and unscoped instead of once per page and per profile. +pub(crate) const CONSENT_STUB_WARNING: &str = "consent_stub_active: audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution"; + /// A user-visible phase reached while collecting browser audit evidence. #[derive(Debug, Clone, Copy)] pub(crate) enum CollectionProgress<'a> { @@ -46,6 +52,10 @@ pub(crate) enum ControlFlow { /// Collect the next target. Continue, /// Stop the crawl without an error (budget reached, challenge rate exceeded). + /// + /// What this can prevent depends on the collector: a sequential one loads no + /// further pages, while the browser collector has already finished + /// navigating by the time it folds, so there it only stops the fold. Stop, } @@ -115,7 +125,17 @@ pub(crate) trait AuditCollector { total: None, url: root, })?; - let root_page = self.collect_page(root, cookies)?; + // A root failure is reported through `on_page` rather than returned, so + // the caller sees the reason as a per-page note exactly as it does from + // the browser collector. With no root page there is nothing to plan + // from, so the crawl ends here. + let root_page = match self.collect_page(root, cookies) { + Ok(page) => page, + Err(error) => { + on_page(root, Err(error))?; + return Ok(()); + } + }; on_progress(CollectionProgress::Planning)?; let targets = planner(root, &root_page)?; if on_page(root, Ok(root_page))? == ControlFlow::Stop { diff --git a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs index 3f416278f..76d57cf9e 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs @@ -52,6 +52,28 @@ const NON_PAGE_EXTENSIONS: &[&str] = &[ ".xml", ".pdf", ".zip", ".mp4", ".mp3", ".rss", ]; +/// ISO 639-1 alpha-2 language codes, sorted for binary search. +/// +/// Country codes are deliberately absent: `/us` and `/tv` are section roots on +/// plenty of publishers, and only the language form appears as a URL locale +/// prefix on its own. +const ISO_639_1_CODES: &[&str] = &[ + "aa", "ab", "ae", "af", "ak", "am", "an", "ar", "as", "av", "ay", "az", "ba", "be", "bg", "bh", + "bi", "bm", "bn", "bo", "br", "bs", "ca", "ce", "ch", "co", "cr", "cs", "cu", "cv", "cy", "da", + "de", "dv", "dz", "ee", "el", "en", "eo", "es", "et", "eu", "fa", "ff", "fi", "fj", "fo", "fr", + "fy", "ga", "gd", "gl", "gn", "gu", "gv", "ha", "he", "hi", "ho", "hr", "ht", "hu", "hy", "hz", + "ia", "id", "ie", "ig", "ii", "ik", "io", "is", "it", "iu", "ja", "jv", "ka", "kg", "ki", "kj", + "kk", "kl", "km", "kn", "ko", "kr", "ks", "ku", "kv", "kw", "ky", "la", "lb", "lg", "li", "ln", + "lo", "lt", "lu", "lv", "mg", "mh", "mi", "mk", "ml", "mn", "mr", "ms", "mt", "my", "na", "nb", + "nd", "ne", "ng", "nl", "nn", "no", "nr", "nv", "ny", "oc", "oj", "om", "or", "os", "pa", "pi", + "pl", "ps", "pt", "qu", "rm", "rn", "ro", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", + "sl", "sm", "sn", "so", "sq", "sr", "ss", "st", "su", "sv", "sw", "ta", "te", "tg", "th", "ti", + "tk", "tl", "tn", "to", "tr", "ts", "tt", "tw", "ty", "ug", "uk", "ur", "uz", "ve", "vi", "vo", + "wa", "wo", "xh", "yi", "yo", "za", "zh", "zu", +]; + +/// Filenames that name a directory's index document rather than a page of their +/// own, so a link to one is treated as a link to the parent directory. const DIRECTORY_INDEX_NAMES: &[&str] = &[ "index.html", "index.htm", @@ -297,13 +319,21 @@ fn same_origin_page_url(root: &Url, raw: &str, section_segment: usize) -> Option { return None; } + // A section reachable only through its index document is still that section: + // `/news/index.html` is `/news`. Rejecting the URL outright loses the + // section; dropping the filename keeps it. + if path + .split('/') + .rfind(|part| !part.is_empty()) + .is_some_and(|last| DIRECTORY_INDEX_NAMES.contains(&last)) + { + url.path_segments_mut().ok()?.pop(); + } + let path = percent_decode_for_filtering(url.path()).to_ascii_lowercase(); let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); if segments.is_empty() { return None; } - if DIRECTORY_INDEX_NAMES.contains(&segments.last().copied().unwrap_or_default()) { - return None; - } if NOISE_SEGMENTS.contains(&segments.get(section_segment).copied().unwrap_or_default()) { return None; } @@ -334,6 +364,8 @@ fn section_at(url: &Url, index: usize) -> Option { .map(str::to_ascii_lowercase) } +/// Whether the requested root is nothing but a locale prefix, which puts +/// sections one segment deeper than usual. fn root_is_locale_prefix(root: &Url) -> bool { let segments: Vec<&str> = root .path() @@ -343,14 +375,27 @@ fn root_is_locale_prefix(root: &Url) -> bool { matches!(segments.as_slice(), [locale] if is_locale_segment(locale)) } +/// Whether a root's single path segment is a locale prefix (`/en`, `/en-gb`) +/// rather than a content section. +/// +/// The language half must be a real ISO 639-1 code. Accepting any two letters +/// read ordinary section roots — `/tv`, `/ai`, `/us` — as locales, which shifts +/// `section_segment` by one: article slugs then become "sections" and the +/// containment check below discards the root's real siblings. fn is_locale_segment(segment: &str) -> bool { - let bytes = segment.as_bytes(); - matches!(bytes, [a, b] if a.is_ascii_alphabetic() && b.is_ascii_alphabetic()) - || matches!(bytes, [a, b, b'-', c, d] - if a.is_ascii_alphabetic() - && b.is_ascii_alphabetic() - && c.is_ascii_alphabetic() - && d.is_ascii_alphabetic()) + let segment = segment.to_ascii_lowercase(); + match segment.as_bytes() { + [_, _] => is_language_code(&segment), + [_, _, b'-', c, d] => { + is_language_code(&segment[..2]) && c.is_ascii_alphabetic() && d.is_ascii_alphabetic() + } + _ => false, + } +} + +/// Whether `segment` is an ISO 639-1 alpha-2 language code. +fn is_language_code(segment: &str) -> bool { + ISO_639_1_CODES.binary_search(&segment).is_ok() } /// Decodes percent escapes solely for normalized path classification. @@ -430,7 +475,11 @@ mod tests { CrawlBudget::default(), ); - assert_eq!(segments(&plan), ["news"]); + assert_eq!( + segments(&plan), + ["news"], + "only the witnessed section should be planned" + ); let section = &plan.sections[0]; assert_eq!( section.landing.as_ref().map(Url::as_str), @@ -498,7 +547,11 @@ mod tests { CrawlBudget::default(), ); - assert_eq!(segments(&plan), ["news"]); + assert_eq!( + segments(&plan), + ["news"], + "only the witnessed section should be planned" + ); assert_eq!( plan.sections[0].landing.as_ref().map(Url::as_str), Some("https://publisher.example/news"), @@ -544,7 +597,11 @@ mod tests { ); assert_eq!(plan.sections.len(), 2, "section cap should be honoured"); - assert_eq!(plan.dropped_sections.len(), 2); + assert_eq!( + plan.dropped_sections.len(), + 2, + "sections past the budget should be reported as dropped" + ); assert!( plan.notes .iter() @@ -576,7 +633,11 @@ mod tests { 2, "root + 2 pages fills max_pages = 3" ); - assert_eq!(plan.dropped_sections.len(), 1); + assert_eq!( + plan.dropped_sections.len(), + 1, + "the section past the budget should be reported as dropped" + ); } #[test] @@ -602,8 +663,14 @@ mod tests { fn empty_input_plans_nothing_rather_than_panicking() { let plan = plan_crawl(&root(), &[], &[], CrawlBudget::default()); - assert!(plan.sections.is_empty()); - assert!(plan.targets().is_empty()); + assert!( + plan.sections.is_empty(), + "no input means no sections to sample" + ); + assert!( + plan.targets().is_empty(), + "no sections means nothing to load" + ); } #[test] @@ -619,9 +686,16 @@ mod tests { CrawlBudget::default(), ); - assert_eq!(plan.section_segment, 1); + assert_eq!( + plan.section_segment, 1, + "a locale root puts sections one segment deeper" + ); assert_eq!(segments(&plan), ["deals", "news"]); - assert_eq!(plan.targets().len(), 4); + assert_eq!( + plan.targets().len(), + 4, + "each section contributes a landing page and an article" + ); } #[test] @@ -646,6 +720,86 @@ mod tests { ); } + #[test] + fn a_two_letter_section_root_is_not_read_as_a_locale() { + // `/tv`, `/ai` and `/us` are section roots, not locales. Reading them as + // locales moves the section segment to 1, so article slugs become + // "sections" and the root's real siblings are discarded. + for root_path in ["/tv", "/ai", "/us"] { + let section_root = Url::parse(&format!("https://publisher.example{root_path}")) + .expect("should parse root"); + let plan = plan_crawl( + §ion_root, + &[ + nav(&format!("{root_path}/story-one")), + nav(&format!("{root_path}/story-two")), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 0, + "`{root_path}` should be a section root, not a locale prefix" + ); + assert_eq!( + segments(&plan), + [root_path.trim_start_matches('/')], + "articles below `{root_path}` should stay one section" + ); + } + } + + #[test] + fn a_real_language_prefix_is_still_read_as_a_locale() { + for root_path in ["/en", "/fr", "/pt-br"] { + let locale_root = Url::parse(&format!("https://publisher.example{root_path}")) + .expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav(&format!("{root_path}/news"))], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 1, + "`{root_path}` is a locale prefix, so sections start one segment in" + ); + assert_eq!(segments(&plan), ["news"]); + } + } + + #[test] + fn a_section_reachable_only_by_its_index_document_collapses_to_the_parent() { + let plan = plan_crawl( + &root(), + &[ + nav("/news/index.html"), + nav("/deals/index.php"), + nav("/sport/home.htm"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["deals", "news", "sport"], + "an index document names its section rather than disqualifying it" + ); + let targets: Vec = plan + .targets() + .iter() + .map(|url| url.path().to_string()) + .collect(); + assert_eq!( + targets, + ["/deals", "/news", "/sport"], + "the parent directory is what gets loaded" + ); + } + #[test] fn locale_root_rejects_candidates_outside_its_path_prefix() { let locale_root = Url::parse("https://publisher.example/en").expect("should parse root"); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs index 78fd6d8a6..fbf889127 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -132,6 +132,13 @@ pub(super) struct EvidenceTable { empty_pages: BTreeSet, /// Page paths that produced slot evidence on at least one selected profile. non_empty_pages: BTreeSet, + /// Div stems any page refused as ambiguous, unioned across the crawl. + /// + /// The verdict has to outlive the page that reached it. Article pages carry + /// several in-content units and refuse the shared prefix; a landing page + /// carries one and would otherwise contribute it as a usable slot, so the + /// written config would depend on which pages the crawl happened to sample. + ambiguous_stems: BTreeSet, } impl EvidenceTable { @@ -153,6 +160,8 @@ impl EvidenceTable { } self.non_empty_pages.insert(path.to_string()); self.empty_pages.remove(path); + self.ambiguous_stems + .extend(discovered.ambiguous_stems.iter().cloned()); for slot in &discovered.slots { let entry = self.slots.entry(slot.div_id.clone()).or_insert_with(|| { @@ -176,16 +185,17 @@ impl EvidenceTable { } } - /// Slots in first-seen order. + /// Slots in first-seen order, excluding stems any page refused as ambiguous. pub(super) fn slots(&self) -> impl Iterator { self.order .iter() + .filter(|div_id| !self.ambiguous_stems.contains(*div_id)) .filter_map(|div_id| self.slots.get(div_id)) } - /// Number of distinct slots observed. + /// Number of usable distinct slots observed. pub(super) fn slot_count(&self) -> usize { - self.slots.len() + self.slots().count() } /// Every page path folded in, whether or not it yielded slots. @@ -202,7 +212,11 @@ impl EvidenceTable { &self.empty_pages } - /// Whether any slot was observed at all. + /// Whether any slot was observed at all, ambiguous ones included. + /// + /// Deliberately not `slot_count() == 0`: a crawl that saw only ambiguous + /// placements did observe an ad stack, and the caller distinguishes "this + /// page has no slots" from "every slot found was refused". pub(super) fn is_empty(&self) -> bool { self.slots.is_empty() } @@ -399,13 +413,16 @@ mod tests { #[test] fn one_placement_under_per_render_div_ids_is_detected() { - // A timestamped token means each page yields a new key - // for the same placement. Same unit, same formats, never co-occurring. + // Each page yields a new key for the same placement: same unit, same + // formats, never co-occurring. The tokens here deliberately do *not* + // match the digit-led shape `discover_gpt_slots` refuses on sight, so + // this exercises the evidence-based detector that catches the stacks + // whose token shape cannot be recognized from one observation. let mut table = EvidenceTable::default(); for (path, div) in [ - ("/features/a", "ex_slot_26329268ce6Bj0uc8sL0_overlay_1"), - ("/news/b", "ex_slot_26329269aoYmv4RQyN3n_overlay_1"), - ("/deals/c", "ex_slot_26329270mYPDB3tz8cpB_overlay_1"), + ("/features/a", "ex_slot_ce6Bj0uc8sL0aa_overlay_1"), + ("/news/b", "ex_slot_aoYmv4RQyN3nbb_overlay_1"), + ("/deals/c", "ex_slot_mYPDB3tz8cpBcc_overlay_1"), ] { table.fold_page( path, @@ -425,6 +442,59 @@ mod tests { ); } + #[test] + fn an_ambiguous_stem_stays_refused_on_every_page() { + // The article page carries two in-content units and refuses the shared + // prefix; the landing page carries one. Folding the landing page must + // not resurrect a prefix that cannot resolve to one element site-wide. + let mut table = EvidenceTable::default(); + table.fold_page( + "/news/story", + &page( + &[ + ( + "/123/site/news", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + ( + "/123/site/news", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-1", + &[(300, 250)], + ), + ], + false, + ), + ); + table.fold_page( + "/", + &page( + &[( + "/123/site/home", + "ad-in_content-1c0de08e5a2f4d6f9b3a7e5c8d1f2a4b-in_content-0", + &[(300, 250)], + )], + false, + ), + ); + + assert_eq!( + table.slots().count(), + 0, + "a stem refused on one page must stay refused, got {:?}", + table.slots().map(|slot| &slot.div_id).collect::>() + ); + assert_eq!( + table.slot_count(), + 0, + "the count should match what is written" + ); + assert!( + !table.is_empty(), + "the crawl did observe an ad stack, so this is not an empty result" + ); + } + #[test] fn genuine_siblings_on_one_unit_are_not_treated_as_fragments() { // Two real in-content positions can share a unit path and formats. What diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index 03837a136..ba7a41735 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -46,11 +46,16 @@ static UUID_SEGMENT: LazyLock = LazyLock::new(|| { /// both breaks runtime div matching and starves template inference of the /// repeated observations it needs. /// -/// The uppercase form is distinctive enough to match bare. The lowercase one is -/// anchored (`_r_`, a short alphanumeric run, `_`) so an ordinary id that merely -/// contains `_r_` keeps its full stem. -static REACT_USE_ID: LazyLock = - LazyLock::new(|| Regex::new(r"_R_|_r_[0-9a-z]{1,8}_").expect("should compile react id regex")); +/// The uppercase form is distinctive enough to match bare, and its hash is +/// included so the match spans the whole ephemeral token — [`normalize_div_stem`] +/// only reads the match *start*, but [`ephemeral_marker_residue`] excises the +/// match, and a residue that still carried the hash would make two renders of one +/// element look like two elements. The lowercase form is anchored (`_r_`, a short +/// alphanumeric run, `_`) so an ordinary id that merely contains `_r_` keeps its +/// full stem. +static REACT_USE_ID: LazyLock = LazyLock::new(|| { + Regex::new(r"_R_[0-9a-z]*_?|_r_[0-9a-z]{1,8}_").expect("should compile react id regex") +}); /// Hosts that serve GPT `gampad/ads` requests. const GAMPAD_HOSTS: &[&str] = &["securepubads.g.doubleclick.net", "pubads.g.doubleclick.net"]; @@ -58,10 +63,6 @@ const GAMPAD_HOSTS: &[&str] = &["securepubads.g.doubleclick.net", "pubads.g.doub /// Common GPT div-id prefix stripped when deriving a slot id. const GPT_DIV_PREFIX: &str = "div-gpt-ad-"; -/// Publisher integration whose div IDs include a per-render timestamp/random -/// token before the placement kind (for example, `_ei_inarticle_1`). -const RH_GAM_KSO_PREFIX: &str = "rh-gam-kso"; - /// Minimum width/height for a format to be treated as a real creative size. /// /// GPT encodes fluid/native aspect-ratio markers (e.g. `4x1`, `8x1`) alongside @@ -94,6 +95,12 @@ pub(crate) struct DiscoveredSlots { pub(crate) had_slot_evidence: bool, /// The reconstructed slots, deduplicated by div id in first-seen order. pub(crate) slots: Vec, + /// Div stems refused because several live elements normalized onto them. + /// + /// Carried separately from `slots` because the verdict is a property of the + /// *site*, not of this page: another page that happens to render only one + /// member of the group must not resurrect the ambiguous prefix. + pub(crate) ambiguous_stems: BTreeSet, /// Diagnostics for placements whose normalized stable stems collided. pub(crate) warnings: Vec, } @@ -115,9 +122,14 @@ pub(crate) fn discover_gpt_slots( ) -> DiscoveredSlots { let mut slots = Vec::new(); let mut warnings = Vec::new(); + let mut ambiguous_stems = BTreeSet::new(); let mut gam_network_id = None; let mut had_slot_evidence = false; - let mut registry_divs: BTreeMap> = BTreeMap::new(); + let mut registry_residues: BTreeMap> = BTreeMap::new(); + // Stems refused outright, so the request fallback cannot re-add them. Kept + // apart from `registry_residues` so a later registry entry cannot read a + // refused stem as a one-member collision group. + let mut refused_stems: BTreeSet = BTreeSet::new(); for entry in registry { let Some(slot) = slot_from_registry(entry, page_has_prebid) else { @@ -127,23 +139,25 @@ pub(crate) fn discover_gpt_slots( if gam_network_id.is_none() { gam_network_id = network_id_from_unit_path(&entry.gam_unit_path); } - if let Some(prefix) = known_per_render_div_prefix(&entry.div_id) { - registry_divs - .entry(slot.div_id.clone()) - .or_default() - .insert(entry.div_id.clone()); - push_unique_warning(&mut warnings, known_per_render_warning(prefix)); + if let Some(prefix) = volatile_prefix_before_placement(&entry.div_id) { + refused_stems.insert(slot.div_id.clone()); + push_unique_warning(&mut warnings, volatile_prefix_warning(&prefix)); continue; } if let Some(prefix) = - push_slot_refusing_collisions(&mut slots, &mut registry_divs, slot, &entry.div_id) + push_slot_refusing_collisions(&mut slots, &mut registry_residues, slot, &entry.div_id) { warnings.push(ambiguous_collision_warning(&prefix)); + ambiguous_stems.insert(prefix); } } - let registry_stems: BTreeSet = registry_divs.keys().cloned().collect(); - let mut request_divs: BTreeMap> = BTreeMap::new(); + let registry_stems: BTreeSet = registry_residues + .keys() + .cloned() + .chain(refused_stems) + .collect(); + let mut request_residues: BTreeMap> = BTreeMap::new(); for request in requests { let Some((network_id, slot, raw_div)) = parse_gampad_request(&request.url) else { continue; @@ -155,14 +169,15 @@ pub(crate) fn discover_gpt_slots( if registry_stems.contains(&slot.div_id) { continue; } - if let Some(prefix) = known_per_render_div_prefix(&raw_div) { - push_unique_warning(&mut warnings, known_per_render_warning(prefix)); + if let Some(prefix) = volatile_prefix_before_placement(&raw_div) { + push_unique_warning(&mut warnings, volatile_prefix_warning(&prefix)); continue; } if let Some(prefix) = - push_slot_refusing_collisions(&mut slots, &mut request_divs, slot, &raw_div) + push_slot_refusing_collisions(&mut slots, &mut request_residues, slot, &raw_div) { warnings.push(ambiguous_collision_warning(&prefix)); + ambiguous_stems.insert(prefix); } } make_slot_ids_unique(&mut slots); @@ -171,33 +186,39 @@ pub(crate) fn discover_gpt_slots( gam_network_id, had_slot_evidence, slots, + ambiguous_stems, warnings, } } -/// Adds one source-local slot unless distinct raw div IDs share its stable stem. +/// Adds one source-local slot unless two distinct *elements* share its stem. +/// +/// Sharing a stem is not by itself ambiguity: one element re-rendered under a +/// fresh framework token is exactly what normalization exists to absorb, and it +/// produces two raw ids that collapse onto one stem. Ambiguity is two elements, +/// which [`ephemeral_marker_residue`] separates from two renders of one. /// -/// The first distinct collision removes the tentatively accepted slot and -/// returns its stem for one diagnostic. Repeats and later collision members stay +/// The first distinct residue removes the tentatively accepted slot and returns +/// its stem for one diagnostic. Repeats and later collision members stay /// suppressed and return `None`. fn push_slot_refusing_collisions( slots: &mut Vec, - seen_divs: &mut BTreeMap>, + seen_residues: &mut BTreeMap>, slot: DiscoveredSlot, raw_div: &str, ) -> Option { let normalized = slot.div_id.clone(); - let raw_div = raw_div.strip_suffix("-container").unwrap_or(raw_div); - match seen_divs.get_mut(&normalized) { + let residue = ephemeral_marker_residue(raw_div); + match seen_residues.get_mut(&normalized) { None => { - seen_divs.insert(normalized, BTreeSet::from([raw_div.to_string()])); + seen_residues.insert(normalized, BTreeSet::from([residue])); slots.push(slot); None } - Some(raw_divs) if raw_divs.contains(raw_div) => None, - Some(raw_divs) => { - let became_ambiguous = raw_divs.len() == 1; - raw_divs.insert(raw_div.to_string()); + Some(residues) if residues.contains(&residue) => None, + Some(residues) => { + let became_ambiguous = residues.len() == 1; + residues.insert(residue); if became_ambiguous { slots.retain(|entry| entry.div_id != normalized); Some(normalized) @@ -208,6 +229,7 @@ fn push_slot_refusing_collisions( } } +/// Operator-facing text for a stem several live elements normalized onto. fn ambiguous_collision_warning(prefix: &str) -> String { format!( "skipped ambiguous div-id prefix `{prefix}`: multiple active elements normalized to it, \ @@ -217,30 +239,64 @@ fn ambiguous_collision_warning(prefix: &str) -> String { ) } -fn known_per_render_div_prefix(div_id: &str) -> Option<&'static str> { +/// The stable prefix of a div id whose per-render token precedes more of the id. +/// +/// Some ad stacks build ids as `__` — a +/// millisecond timestamp plus a random suffix sitting *before* the part that +/// distinguishes one placement from the next. Such an id can be written neither +/// literally (the token changes on the next render) nor as a prefix: the only +/// stable prefix stops at the token, and that prefix reaches every placement in +/// the family, while the runtime resolves a prefix to a single element. So the +/// slot is refused from a single observation, without waiting for a second +/// placement to prove the collision. +/// +/// The shape decides, not the vendor: any segment that is a long digit run +/// followed by more alphanumerics counts, so a new stack with the same layout +/// needs no code change. A token in *trailing* position is deliberately not this +/// case — everything before it still identifies the element — and is left to +/// normalization and the same-page collision check. +fn volatile_prefix_before_placement(div_id: &str) -> Option { let div_id = div_id.strip_suffix("-container").unwrap_or(div_id); - let remainder = div_id.strip_prefix("rh-gam-kso_")?; - let (token, placement) = remainder.split_once("_ei_")?; - let leading_digits = token.bytes().take_while(u8::is_ascii_digit).count(); - let token_is_dynamic = leading_digits >= 8 - && token.len() > leading_digits - && token.bytes().all(|byte| byte.is_ascii_alphanumeric()); - let placement_index = placement - .strip_prefix("inarticle_") - .or_else(|| placement.strip_prefix("overlay_"))?; - let placement_is_known = - !placement_index.is_empty() && placement_index.bytes().all(|byte| byte.is_ascii_digit()); - (token_is_dynamic && placement_is_known).then_some(RH_GAM_KSO_PREFIX) + let mut start = 0_usize; + for (index, character) in div_id.char_indices() { + if character != '_' && character != '-' { + continue; + } + if is_per_render_token(&div_id[start..index]) { + let prefix = div_id[..start].trim_end_matches(['_', '-']); + // A delimiter is one byte, so the remainder starts just past it. + return (!prefix.is_empty() && !div_id[index + 1..].is_empty()) + .then(|| prefix.to_string()); + } + start = index + character.len_utf8(); + } + None } -fn known_per_render_warning(prefix: &str) -> String { +/// Whether one div-id segment is a per-render token: a long leading digit run (a +/// millisecond timestamp) followed by more alphanumerics (a random suffix). +/// +/// Both halves are required. A bare digit run is how publishers write stable +/// placement indices, and a token with a non-alphanumeric character is some +/// other structure than a generated id. +fn is_per_render_token(segment: &str) -> bool { + let leading_digits = segment.bytes().take_while(u8::is_ascii_digit).count(); + leading_digits >= 8 + && segment.len() > leading_digits + && segment.bytes().all(|byte| byte.is_ascii_alphanumeric()) +} + +/// Operator-facing text for a div-id family carrying a per-render token. +fn volatile_prefix_warning(prefix: &str) -> String { format!( - "skipped known per-render div-id family `{prefix}`: exact div ids change across renders \ - and no distinct stable element prefix is available; expose distinct stable div ids in \ - publisher markup before configuring these placements" + "skipped volatile div-id family `{prefix}`: a per-render token sits before the placement \ + suffix, so exact div ids change across renders and no distinct stable element prefix is \ + available; expose distinct stable div ids in publisher markup before configuring these \ + placements" ) } +/// Records `warning` unless the same text was already recorded for this page. fn push_unique_warning(warnings: &mut Vec, warning: String) { if !warnings.contains(&warning) { warnings.push(warning); @@ -314,23 +370,62 @@ fn is_usable_unit_path(path: &str) -> bool { /// → `ad-in_content`. fn normalize_div_stem(div_id: &str) -> String { let stem = div_id.strip_suffix("-container").unwrap_or(div_id); - let mut cut = stem.len(); - if let Some(matched) = REACT_USE_ID.find(stem) { - cut = cut.min(matched.start()); - } - let uuid = UUID_SEGMENT.find(stem); - let hex = HEX_HASH_SEGMENT.find_iter(stem).find(|matched| { - matched - .as_str() - .bytes() - .any(|byte| matches!(byte, b'a'..=b'f')) - }); - if let Some(matched) = uuid.into_iter().chain(hex).min_by_key(regex::Match::start) { - cut = cut.min(matched.start()); - } + let cut = ephemeral_marker_ranges(stem) + .first() + .map_or(stem.len(), |range| range.start); stem[..cut].trim_end_matches('-').to_string() } +/// Byte ranges of every ephemeral per-render marker in `stem`, in order and +/// without overlaps. +/// +/// A hex-hash candidate must contain at least one `a`-`f`; a run of 16+ digits +/// is how publishers write stable ids, not a hash. +fn ephemeral_marker_ranges(stem: &str) -> Vec> { + let mut ranges: Vec> = REACT_USE_ID + .find_iter(stem) + .chain(UUID_SEGMENT.find_iter(stem)) + .chain(HEX_HASH_SEGMENT.find_iter(stem).filter(|matched| { + matched + .as_str() + .bytes() + .any(|byte| matches!(byte, b'a'..=b'f')) + })) + .map(|matched| matched.range()) + .collect(); + ranges.sort_by_key(|range| range.start); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + match merged.last_mut() { + Some(last) if range.start < last.end => last.end = last.end.max(range.end), + _ => merged.push(range), + } + } + merged +} + +/// The parts of a raw div id that no ephemeral marker covered, NUL-joined. +/// +/// [`normalize_div_stem`] truncates at the first marker, so two ids differing +/// only *inside* a marker collapse onto one stem — the signature of one element +/// re-rendered. What the markers did not cover separates that from two elements: +/// `ad-header-0-_R_3f_` and `ad-header-0-_r_0_` leave the same residue (one +/// element, two renders), while `…-in_content-0` and `…-in_content-1` do not +/// (two siblings). A live div id cannot contain NUL, so joining on it cannot +/// make two different residues compare equal. +fn ephemeral_marker_residue(div_id: &str) -> String { + let stem = div_id.strip_suffix("-container").unwrap_or(div_id); + let mut residue = String::with_capacity(stem.len()); + let mut previous = 0_usize; + for range in ephemeral_marker_ranges(stem) { + residue.push_str(&stem[previous..range.start]); + residue.push('\0'); + previous = range.end; + } + residue.push_str(&stem[previous..]); + residue +} + /// Extracts the leading network id from a GAM ad-unit path (`//...`). fn network_id_from_unit_path(path: &str) -> Option { let segment = path.trim_start_matches('/').split('/').next()?; @@ -962,11 +1057,18 @@ mod tests { "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cnews%2Catf&dids=ad-a%2Cad-b&prev_iu_szs=300x250", )]); - assert!(discovered.slots.is_empty()); + assert!( + discovered.slots.is_empty(), + "a comma-joined SRA did list is not one element" + ); } #[test] - fn same_page_hex_normalization_collision_is_refused() { + fn one_element_under_two_render_tokens_is_not_a_collision() { + // Both ids describe in-content placement 0; only the hash between the + // two copies of the placement name differs, which is what one element + // re-rendered looks like. Refusing here would refuse the very shape + // normalization exists to absorb. let registry = vec![ registry_slot( "/987654321/site/homepage", @@ -982,12 +1084,75 @@ mod tests { let discovered = discover_gpt_slots(®istry, &[], false); - assert!(discovered.had_slot_evidence); + assert_eq!( + discovered.slots.len(), + 1, + "two renders of one element are one slot, got {:?}", + discovered.slots + ); + assert_eq!(discovered.slots[0].div_id, "ad-in_content"); + assert!( + discovered.warnings.is_empty(), + "a re-render is not an ambiguity to report, got {:?}", + discovered.warnings + ); + assert!(discovered.ambiguous_stems.is_empty()); + } + + #[test] + fn sibling_placements_sharing_one_stem_are_refused() { + // Same shape as above, but the trailing placement index differs: these + // are two live elements, and one prefix cannot resolve to both. + let registry = vec![ + registry_slot( + "/987654321/site/homepage", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + registry_slot( + "/987654321/site/homepage", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-1", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); assert!( discovered.slots.is_empty(), "neither a broad prefix nor per-render exact IDs are safe" ); assert_ambiguous_collision_warning(&discovered, "ad-in_content"); + assert!( + discovered.ambiguous_stems.contains("ad-in_content"), + "the verdict must travel with the evidence, got {:?}", + discovered.ambiguous_stems + ); + } + + #[test] + fn react_server_and_client_render_tokens_are_one_slot() { + // A hydrating publisher reports the SSR id and the client id for the + // same element. Both must collapse rather than refuse each other. + let registry = vec![ + registry_slot("/123456789/site/news", "ad-header-0-_R_3f_", &[(728, 90)]), + registry_slot("/123456789/site/news", "ad-header-0-_r_0_", &[(728, 90)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "SSR and client renders of one element are one slot, got {:?}", + discovered.slots + ); + assert_eq!(discovered.slots[0].div_id, "ad-header-0"); + assert!(discovered.warnings.is_empty()); } #[test] @@ -1005,7 +1170,10 @@ mod tests { let discovered = discover_gpt_slots(®istry, &[], false); - assert!(discovered.had_slot_evidence); + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); assert!( discovered.slots.is_empty(), "no repeat or later collision member may resurrect the group" @@ -1024,68 +1192,107 @@ mod tests { ), ]); - assert!(discovered.had_slot_evidence); - assert!(discovered.slots.is_empty()); - assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "a refused placement must not be written, got {:?}", + discovered.slots + ); + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "refusing a slot must not discard the network id" + ); assert_ambiguous_collision_warning(&discovered, "ad-x"); } #[test] - fn single_known_per_render_registry_slot_is_refused() { + fn single_volatile_family_registry_slot_is_refused() { let discovered = discover_gpt_slots( &[registry_slot( "/123456789/site_in-article_desktop_1", - "rh-gam-kso_12345678AbCdEfGh_ei_inarticle_1", + "vendor-tag_12345678AbCdEfGh_slot_inarticle_1", &[(300, 250)], )], &[], false, ); - assert!(discovered.had_slot_evidence); + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); assert!( discovered.slots.is_empty(), - "one observation of a known per-render family must not be written literally" + "one observation of a per-render family must not be written literally" ); assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); - assert_known_per_render_warning(&discovered); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); } #[test] - fn single_known_per_render_request_slot_is_refused() { + fn single_volatile_family_request_slot_is_refused() { let discovered = from_requests(&[request( - "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite_in-article_desktop_1&dids=rh-gam-kso_12345678AbCdEfGh_ei_inarticle_1&prev_iu_szs=300x250", + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite_in-article_desktop_1&dids=vendor-tag_12345678AbCdEfGh_slot_inarticle_1&prev_iu_szs=300x250", )]); - assert!(discovered.had_slot_evidence); - assert!(discovered.slots.is_empty()); - assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); - assert_known_per_render_warning(&discovered); - } - - #[test] - fn known_per_render_match_does_not_claim_arbitrary_vendor_ids() { - assert_eq!( - known_per_render_div_prefix("rh-gam-kso_12345678AbCdEfGh_ei_inarticle_1"), - Some("rh-gam-kso") + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "a refused placement must not be written, got {:?}", + discovered.slots ); assert_eq!( - known_per_render_div_prefix("rh-gam-kso_12345678AbCdEfGh_ei_overlay_1-container"), - Some("rh-gam-kso") + discovered.gam_network_id.as_deref(), + Some("123456789"), + "refusing a slot must not discard the network id" ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn volatile_prefix_covers_every_placement_after_the_token() { + // The token's position is what makes the id unusable, so the placement + // that follows it is irrelevant: every one of these leaves `vendor-tag` + // as the only stable prefix, and that prefix reaches all of them. + for volatile in [ + "vendor-tag_12345678AbCdEfGh_slot_inarticle_1", + "vendor-tag_12345678AbCdEfGh_slot_overlay_1-container", + "vendor-tag_12345678AbCdEfGh_slot_sidebar_1", + "vendor-tag_12345678AbCdEfGh_slot_overlay_stable", + "vendor-tag_12345678AbCdEfGh_slot_overlay_1_extra", + ] { + assert_eq!( + volatile_prefix_before_placement(volatile).as_deref(), + Some("vendor-tag"), + "`{volatile}` should be refused as a volatile family" + ); + } + } + + #[test] + fn volatile_prefix_does_not_claim_stable_div_ids() { for stable in [ - "rh-gam-kso_stable_ei_inarticle_1", - "rh-gam-kso_12345678_ei_inarticle_1", - "rh-gam-kso_12345678AbCdEfGh_ei_sidebar_1", - "rh-gam-kso_12345678AbCdEfGh_ei_overlay_stable", - "rh-gam-kso_12345678AbCdEfGh_ei_overlay_", - "rh-gam-kso_12345678AbCdEfGh_ei_overlay_1_extra", - "rh-gam-kso-header", + // No per-render token at all. + "vendor-tag_stable_slot_inarticle_1", + // A bare digit run is how stable placement indices are written. + "vendor-tag_12345678_slot_inarticle_1", + "ad-slot-1234567890123456-tail", + // The token is trailing, so the prefix before it still identifies + // this element and normalization/collision handling own the case. + "vendor-tag_slot_inarticle_12345678AbCdEfGh", + "vendor-tag-header", ] { assert_eq!( - known_per_render_div_prefix(stable), + volatile_prefix_before_placement(stable), None, - "`{stable}` should not match the narrow per-render family" + "`{stable}` should stay eligible" ); } } @@ -1110,7 +1317,10 @@ mod tests { let discovered = discover_gpt_slots(®istry, &requests, false); - assert!(discovered.had_slot_evidence); + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); assert!( discovered.slots.is_empty(), "request fallback must not resurrect an ambiguous registry stem" @@ -1157,11 +1367,25 @@ mod tests { ); } - fn assert_known_per_render_warning(discovered: &DiscoveredSlots) { - assert_eq!(discovered.warnings.len(), 1); + fn assert_volatile_prefix_warning(discovered: &DiscoveredSlots, prefix: &str) { + assert_eq!( + discovered.warnings.len(), + 1, + "should report the family once, got {:?}", + discovered.warnings + ); let warning = &discovered.warnings[0]; - assert!(warning.contains("rh-gam-kso")); - assert!(warning.contains("change across renders")); - assert!(warning.contains("distinct stable div ids")); + assert!( + warning.contains(prefix), + "warning should name the family prefix, got {warning}" + ); + assert!( + warning.contains("change across renders"), + "warning should explain why the exact ids are unsafe, got {warning}" + ); + assert!( + warning.contains("distinct stable div ids"), + "warning should tell the operator how to make the placements configurable, got {warning}" + ); } } diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index af1c1697c..8354e51c0 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -20,7 +20,8 @@ use trusted_server_core::creative_opportunities::{ }; use url::Url; -use crate::commands::audit::ad_templates::origin_changed; +use crate::commands::audit::ad_templates::{origin_changed, without_fragment}; +use crate::commands::audit::collector::GenerateBrowserOpts; use crate::commands::audit::generate::collector::AuditCollector; use crate::commands::audit::generate::slot_toml::{ render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, toml_string, @@ -98,6 +99,8 @@ pub(crate) struct GenerateArgs { /// cookie) so the origin serves the real page instead of a challenge. #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = crate::commands::audit::parse_cookie)] pub(crate) cookies: Vec<(String, String)>, + #[command(flatten)] + pub(crate) browser: GenerateBrowserOpts, } const DEFAULT_JS_ASSETS_PATH: &str = "js-assets.toml"; @@ -558,9 +561,13 @@ pub(crate) fn run_update_slots( &mut |_, root| { root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); if origin_changed(&target_url, &root_url) { + // Origins only: the origin is what the refusal is about, and + // a full URL would echo any `user:password@` the operator + // passed into stderr. return cli_error(format!( "refusing cross-origin root redirect from {} to {}; the requested origin is the audit and cookie trust boundary", - target_url, root_url + target_url.origin().ascii_serialization(), + root_url.origin().ascii_serialization() )); } let plan = crawl_plan::plan_crawl( @@ -585,7 +592,12 @@ pub(crate) fn run_update_slots( } } Err(error) => { - notes.push(format!("skipped `{url}` on {first_label}: {error}")); + // Path only, like the progress lines: a planned target + // still carries the origin and any userinfo. + notes.push(format!( + "skipped `{}` on {first_label}: {error}", + url.path() + )); } } Ok(collector::ControlFlow::Continue) @@ -601,6 +613,15 @@ pub(crate) fn run_update_slots( )) })?; notes.extend(plan.notes.iter().cloned()); + // Fragments never reach the server, so only a difference the origin acted on + // counts as a redirect worth reporting. + if without_fragment(&root_url) != without_fragment(&target_url) { + notes.push(format!( + "followed a root redirect from `{}` to `{}`; slots and page patterns are derived from the final URL", + target_url.path(), + root_url.path() + )); + } // Every profile walks the same pages into the same table. When two profiles // disagree about a slot's ad-unit path, that shows up as two observations of @@ -743,13 +764,15 @@ pub(crate) fn run_update_slots( if request.dry_run { let old_managed = managed_creative_projection(&existing)?; let new_managed = managed_creative_projection(&updated)?; - let diff = similar::TextDiff::from_lines(&old_managed, &new_managed); if old_managed == new_managed { - writeln!(out, "No managed creative-opportunity changes.").map_err(|error| { + // Stdout is the diff surface, so an English sentence there would + // break a redirected `--dry-run`; an empty diff is the stdout answer. + writeln!(err, "No managed creative-opportunity changes.").map_err(|error| { report_error(format!("failed to write preview output: {error}")) })?; return Ok(()); } + let diff = similar::TextDiff::from_lines(&old_managed, &new_managed); writeln!( out, "{}", @@ -773,6 +796,11 @@ pub(crate) fn run_update_slots( request.config_path.display() )); } + // A writer could still land between this check and the rename below. That + // window is microseconds against a browser crawl's minutes, and the rename + // is atomic, so the loser of the race loses a whole write rather than half + // of one. Closing it properly would need file locking the operator's editor + // does not take part in. write_file_atomically(request.config_path, &updated).map_err(|error| { report_error(format!( "failed to write config {}: {error}", @@ -922,7 +950,17 @@ fn fold_collected( // so this is the complete set, not a second copy. let artifact = analyze_collected_page(collected)?; for warning in &artifact.warnings { - notes.push(format!("`{}`: {warning}", url.path())); + // The consent stub is a property of the run, not of this page. Scoping it + // to a path and repeating it per page and profile buries the per-page + // diagnostics an operator is reading these notes for. + let note = if warning == collector::CONSENT_STUB_WARNING { + warning.clone() + } else { + format!("`{}`: {warning}", url.path()) + }; + if !notes.contains(¬e) { + notes.push(note); + } } if let Some(reason) = looks_like_an_interstitial(&artifact) { notes.push(format!("`{}`: {reason}", url.path())); @@ -995,7 +1033,10 @@ fn crawl_sections( } } Err(error) => { - notes.push(format!("skipped `{url}` on {profile_label}: {error}")); + notes.push(format!( + "skipped `{}` on {profile_label}: {error}", + url.path() + )); } } Ok(collector::ControlFlow::Continue) @@ -1024,6 +1065,13 @@ fn guard_challenge_rate(table: &evidence::EvidenceTable) -> CliResult<()> { )) } +/// Refuses a merge that would reinterpret templated slots the config already has. +/// +/// # Errors +/// +/// Returns an error when preserved `{section}` slots were written against a +/// different section policy than this run inferred, since the merge would leave +/// them pointing at ad units nobody configured. fn validate_merge_policy( existing: Option<&CreativeOpportunitiesConfig>, inferred: Option<&unit_template::SectionPolicy>, @@ -1043,7 +1091,18 @@ fn validate_merge_policy( let Some(inferred) = inferred.filter(|_| preserves_template) else { return Ok(()); }; - let configured_root = existing.section_root.as_deref().unwrap_or_default(); + // A `{section}` slot with no `section_root` cannot load at all — + // `validate_runtime` requires one — so there is no working policy to + // preserve and nothing for the inferred one to contradict. Adopting it is + // what makes such a config loadable, and `check_candidate` still gates the + // result, so this is not the refusal case. + let Some(configured_root) = existing + .section_root + .as_deref() + .filter(|root| !root.is_empty()) + else { + return Ok(()); + }; let configured_segment = existing.section_segment.unwrap_or(0); if configured_root != inferred.section_root || configured_segment != inferred.section_segment { return cli_error(format!( @@ -1531,6 +1590,7 @@ mod tests { no_config: false, force: false, cookies: Vec::new(), + browser: GenerateBrowserOpts::default(), } } @@ -1601,6 +1661,57 @@ mod tests { .expect("replace is an explicit policy migration"); } + #[test] + fn the_consent_stub_note_is_reported_once_and_unscoped() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + for url in [ + "https://publisher.example/", + "https://publisher.example/news", + ] { + let mut page = collected_page(); + page.requested_url = url.to_string(); + page.final_url = url.to_string(); + page.warnings + .push(collector::CONSENT_STUB_WARNING.to_string()); + fold_collected( + &mut table, + &Url::parse(url).expect("should parse fixture URL"), + &page, + &mut notes, + ) + .expect("should fold page evidence"); + } + + assert_eq!( + notes, + [collector::CONSENT_STUB_WARNING.to_string()], + "a run-wide fact should appear once, without a page path" + ); + } + + #[test] + fn merge_adopts_the_inferred_policy_when_none_is_configured() { + // A hand-written `{section}` slot with no `section_root` describes a + // config the runtime refuses to load, so the first merge should repair it + // rather than demand `--replace` (which would discard the hand-tuned + // slots it is preserving). + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let inferred = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + + validate_merge_policy(Some(&existing), Some(&inferred), false) + .expect("an unset section_root is no policy to preserve"); + } + #[test] fn resolve_output_plan_rejects_no_outputs() { let mut args = audit_args("https://publisher.example"); @@ -1660,6 +1771,7 @@ mod tests { no_config: false, force: false, cookies: Vec::new(), + browser: GenerateBrowserOpts::default(), }; let collector = FakeCollector::new(collected_page()); let mut out = Vec::new(); @@ -2057,6 +2169,7 @@ mod tests { collected.requested_url = "http://publisher.example/".to_string(); collected.final_url = "https://publisher.example/".to_string(); let collector = FakeCollector::new(collected); + let mut notes = Vec::new(); run_update_slots( &UpdateSlotsRequest { @@ -2071,7 +2184,7 @@ mod tests { }, &[("desktop", &collector)], &mut std::io::sink(), - &mut std::io::sink(), + &mut notes, ) .expect("a same-host HTTPS upgrade should not be treated as cross-origin"); @@ -2082,6 +2195,11 @@ mod tests { Some("div-gpt-ad-header"), "evidence from the upgraded root should be written" ); + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("followed a root redirect"), + "an accepted redirect should say the run switched URLs, got {notes:?}" + ); } #[test] diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 72f54d417..1c401cb3d 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -578,61 +578,80 @@ fn ensure_only_managed_fields_changed(before: &str, after: &str) -> CliResult<() Ok(()) } -/// Whether `document` uses CRLF line endings (so edits preserve them). -fn uses_crlf(document: &str) -> bool { - let mut multiline: Option = None; +/// Byte offsets of the `\n` bytes that terminate a document line. +/// +/// Only newlines outside comments and string values delimit lines, so the scan +/// skips a `#` comment to end of line, skips single-line basic and literal +/// strings, and tracks multiline `"""` / `'''` bodies. Without the comment and +/// single-line-string cases a stray triple quote desynchronizes the scan and the +/// document's line endings are flipped or left mixed — a rewrite +/// [`ensure_only_managed_fields_changed`] cannot catch, because it compares +/// parsed values. +fn document_newlines(document: &str) -> Vec { let bytes = document.as_bytes(); + let mut newlines = Vec::new(); let mut index = 0_usize; while index < bytes.len() { - if let Some(quote) = multiline { - if bytes[index..].starts_with(&[quote, quote, quote]) { - multiline = None; - index += 3; - continue; + match bytes[index] { + b'#' => { + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + } + b'\n' => { + newlines.push(index); + index += 1; } - } else if bytes[index..].starts_with(b"\"\"\"") { - multiline = Some(b'\"'); - index += 3; - continue; - } else if bytes[index..].starts_with(b"'''") { - multiline = Some(b'\''); - index += 3; - continue; - } else if bytes[index] == b'\n' { - return index > 0 && bytes[index - 1] == b'\r'; + quote @ (b'"' | b'\'') => { + if bytes[index..].starts_with(&[quote, quote, quote]) { + index += 3; + while index < bytes.len() && !bytes[index..].starts_with(&[quote, quote, quote]) + { + index += 1; + } + index = index.saturating_add(3).min(bytes.len()); + } else { + index += 1; + while index < bytes.len() && bytes[index] != quote && bytes[index] != b'\n' { + index += if quote == b'"' && bytes[index] == b'\\' { + 2 + } else { + 1 + }; + } + if index < bytes.len() && bytes[index] == quote { + index += 1; + } + } + } + _ => index += 1, } - index += 1; } - false + newlines } -/// Converts document line terminators while leaving multiline-string content intact. +/// Whether `document` uses CRLF line endings (so edits preserve them). +fn uses_crlf(document: &str) -> bool { + let bytes = document.as_bytes(); + document_newlines(document) + .first() + .is_some_and(|&index| index > 0 && bytes[index - 1] == b'\r') +} + +/// Converts document line terminators while leaving string content intact. fn convert_document_lf_to_crlf(document: &str) -> String { + let bytes = document.as_bytes(); let mut output = String::with_capacity(document.len()); - let mut multiline: Option = None; - let mut chars = document.chars().peekable(); - while let Some(ch) = chars.next() { - if matches!(ch, '\"' | '\'') { - let mut probe = chars.clone(); - if probe.next() == Some(ch) && probe.next() == Some(ch) { - output.push(ch); - output.push(chars.next().expect("should have second quote")); - output.push(chars.next().expect("should have third quote")); - multiline = if multiline == Some(ch) { - None - } else if multiline.is_none() { - Some(ch) - } else { - multiline - }; - continue; - } - } - if ch == '\n' && multiline.is_none() && !output.ends_with('\r') { + let mut previous = 0_usize; + for index in document_newlines(document) { + output.push_str(&document[previous..index]); + if index == 0 || bytes[index - 1] != b'\r' { output.push('\r'); } - output.push(ch); + output.push('\n'); + previous = index + 1; } + output.push_str(&document[previous..]); output } @@ -1167,6 +1186,37 @@ slot_id = "sidebar" ); } + #[test] + fn a_triple_quote_in_a_comment_does_not_desynchronize_the_line_scan() { + // A `"""` inside a comment is not a multiline string. Treating it as one + // makes the rest of the document read as string content, so a CRLF file + // is detected as LF and gets rewritten wholesale. + let existing = "# see \"\"\" docs\r\n[creative_opportunities]\r\n\ + gam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "the document's CRLF endings must survive a triple quote in a comment, got {out:?}" + ); + } + + #[test] + fn a_triple_quote_in_a_single_line_string_does_not_desynchronize_the_line_scan() { + let existing = "[publisher]\r\nlabel = 'a \"\"\" b'\r\n\r\n\ + [creative_opportunities]\r\ngam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "the document's CRLF endings must survive a triple quote in a value, got {out:?}" + ); + } + #[test] fn splice_does_not_rewrite_bare_lf_inside_crlf_multiline_string() { let existing = "[publisher]\r\nother = \"\"\"a\nb\"\"\"\r\n\r\n\ diff --git a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs index ecf34d168..faf8ea67d 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs @@ -384,9 +384,12 @@ fn build_template(slot: &SlotEvidence, varying: usize) -> String { /// Replays `template` through the runtime renderer against every observation. /// -/// This is the gate that catches a section slug the path cannot reproduce — a -/// publisher whose `/site-news` pages request `.../sitenews`, say, where -/// the derived section and the observed segment differ. +/// Defense in depth rather than the primary gate: [`analyse_slot`] already +/// refuses to call a slot templatable when the derived section and the observed +/// segment disagree — a publisher whose `/site-news` pages request +/// `.../sitenews`, say — so a mismatch reaching here would mean inference and +/// the runtime renderer disagree. The template is then dropped instead of +/// written, and the diagnostic names the paths that did not reproduce. fn verify_round_trip( template: &str, slot: &SlotEvidence, diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 715010532..0bda921bc 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -16,6 +16,7 @@ use clap::{Args, Subcommand}; use crate::app_config::AppConfigArgs; use crate::commands::audit::collector::{BrowserOpts, GenerateBrowserOpts}; use crate::commands::audit::page::PageAuditArgs; +use crate::error::{CliResult, cli_error}; use crate::run::RunOutcome; /// Parses and validates an `http`/`https` URL, rejecting all other schemes. @@ -91,6 +92,8 @@ pub(crate) struct LegacyGenerateArgs { requires = "legacy_url" )] pub(crate) cookies: Vec<(String, String)>, + #[command(flatten)] + pub(crate) browser: GenerateBrowserOpts, } /// `ts audit` subcommands. @@ -253,7 +256,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result { let raw_config = std::fs::read_to_string(&app_config_path).map_err(|error| { format!("failed to read {}: {error}", app_config_path.display()) })?; - let existing_creative = best_effort_creative_config(&raw_config); + let existing_creative = creative_config(&raw_config)?; let profiles = gen_args.profiles()?; let collectors: Vec = profiles .iter() @@ -298,18 +301,22 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result { ad_templates::run_verify(verify_args) } Some(AuditSubcommand::Generate(generate_args)) => { + generate_args.browser.validate()?; let stdout = std::io::stdout(); let mut out = stdout.lock(); - let collector = generate::browser_collector::BrowserAuditCollector::default(); + let collector = generate::browser_collector::BrowserAuditCollector::default() + .with_browser_options(&generate_args.browser); generate::run_generate(generate_args, &collector, &mut out) .map(|()| RunOutcome::Success) } None => match args.legacy_url.as_ref() { Some(url) => { + args.legacy_generate.browser.validate()?; let generate_args = legacy_generate_args(args, url); let stdout = std::io::stdout(); let mut out = stdout.lock(); - let collector = generate::browser_collector::BrowserAuditCollector::default(); + let collector = generate::browser_collector::BrowserAuditCollector::default() + .with_browser_options(&generate_args.browser); generate::run_generate(&generate_args, &collector, &mut out) .map(|()| RunOutcome::Success) } @@ -320,13 +327,39 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result { } } -fn best_effort_creative_config( +/// Reads the config's `[creative_opportunities]` section, when it has one. +/// +/// An unrelated invalid setting elsewhere in the document must not hide the +/// section — the runtime rejects such a file, but the operator still has to be +/// able to update slots in it — so the document is read as plain TOML rather +/// than through [`Settings`](trusted_server_core::settings::Settings). +/// +/// A section that is present but unreadable is *not* treated as absent. +/// `CreativeOpportunitiesConfig` uses `deny_unknown_fields`, so one mistyped key +/// would otherwise leave the merge with nothing to merge into and replace the +/// operator's entire slot array. +/// +/// # Errors +/// +/// Returns a user-facing error when the section is present but cannot be +/// deserialized. +fn creative_config( document: &str, -) -> Option { - toml::from_str::(document) +) -> CliResult> { + let Some(section) = toml::from_str::(document) .ok() .and_then(|value| value.get("creative_opportunities").cloned()) - .and_then(|value| value.try_into().ok()) + else { + return Ok(None); + }; + match section.try_into() { + Ok(config) => Ok(Some(config)), + Err(error) => cli_error(format!( + "failed to read the existing `[creative_opportunities]` section, so generating \ + slots would discard the configured ones: {error}. Fix the section (or delete it) \ + and re-run" + )), + } } fn legacy_generate_args(args: &AuditArgs, url: &url::Url) -> generate::GenerateArgs { @@ -338,6 +371,7 @@ fn legacy_generate_args(args: &AuditArgs, url: &url::Url) -> generate::GenerateA no_config: args.legacy_generate.no_config, force: args.legacy_generate.force, cookies: args.legacy_generate.cookies.clone(), + browser: args.legacy_generate.browser.clone(), } } @@ -363,16 +397,50 @@ mod tests { } #[test] - fn invalid_baseline_still_yields_best_effort_creative_config() { + fn invalid_setting_outside_the_section_still_yields_creative_config() { let document = "unknown_runtime_key = true\n\ [creative_opportunities]\ngam_network_id = \"123\"\n"; - let creative = best_effort_creative_config(document) - .expect("an unrelated invalid setting must not hide creative config"); + let creative = creative_config(document) + .expect("an unrelated invalid setting must not hide creative config") + .expect("the section is present"); assert_eq!(creative.gam_network_id, "123"); } + #[test] + fn absent_section_reads_as_absent() { + let creative = + creative_config("[auction]\nenabled = true\n").expect("should read the document"); + + assert!( + creative.is_none(), + "a document with no `[creative_opportunities]` has no configured slots" + ); + } + + #[test] + fn unreadable_section_is_refused_rather_than_read_as_absent() { + // `deny_unknown_fields` makes one mistyped key inside the section fail + // to deserialize. Reading that as "no slots configured" would let a + // merge replace the operator's entire slot array. + let document = "[creative_opportunities]\n\ + gam_network_id = \"123\"\n\ + gam_netwrok_id = \"123\"\n\ + [[creative_opportunities.slot]]\n\ + id = \"header\"\n\ + div_id = \"ad-header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + + let error = creative_config(document).expect_err("should refuse an unreadable section"); + + assert!( + error.contains("would discard the configured ones"), + "error should say what merging would cost, got {error}" + ); + } + #[test] fn parse_cookie_rejects_missing_equals() { let err = parse_cookie("datadome").expect_err("should reject missing `=`"); @@ -402,6 +470,10 @@ mod tests { no_config: false, force: true, cookies: vec![("session".to_string(), "example".to_string())], + browser: GenerateBrowserOpts { + headful: true, + ..GenerateBrowserOpts::default() + }, }, }; @@ -424,5 +496,9 @@ mod tests { generate.cookies, [("session".to_string(), "example".to_string())] ); + assert!( + generate.browser.headful, + "browser flags passed to the legacy form should reach generation" + ); } } diff --git a/crates/trusted-server-cli/src/commands/audit/page.rs b/crates/trusted-server-cli/src/commands/audit/page.rs index c5708f109..9edbcf5d0 100644 --- a/crates/trusted-server-cli/src/commands/audit/page.rs +++ b/crates/trusted-server-cli/src/commands/audit/page.rs @@ -58,9 +58,14 @@ fn run_with_collector( fn write_summary(out: &mut dyn Write, url: &url::Url, page: &CollectedPage) -> Result<(), String> { let to_err = |error: io::Error| format!("failed to write command output: {error}"); writeln!(out, "url: {url}").map_err(to_err)?; - writeln!(out, "final url: {}", page.final_url).map_err(to_err)?; - // The title and collector warning messages are page-controlled, so escape - // control characters before they reach the operator's terminal. + // The final URL, title, and collector warning messages are page-controlled, + // so escape control characters before they reach the operator's terminal. + writeln!( + out, + "final url: {}", + escape_terminal_text(page.final_url.as_str()) + ) + .map_err(to_err)?; writeln!(out, "title: {}", escape_terminal_text(&page.title)).map_err(to_err)?; writeln!(out, "scripts: {}", page.script_count).map_err(to_err)?; writeln!(out, "resources: {}", page.resource_count).map_err(to_err)?; @@ -75,3 +80,73 @@ fn write_summary(out: &mut dyn Write, url: &url::Url, page: &CollectedPage) -> R } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::output::Warning; + + fn collected(final_url: &str, title: &str, warnings: Vec) -> CollectedPage { + CollectedPage { + final_url: url::Url::parse(final_url).expect("should parse fixture URL"), + title: title.to_string(), + script_count: 3, + resource_count: 42, + warnings, + ad_evidence: None, + } + } + + fn summary(page: &CollectedPage, requested: &str) -> String { + let url = url::Url::parse(requested).expect("should parse requested URL"); + let mut out = Vec::new(); + write_summary(&mut out, &url, page).expect("should write summary"); + String::from_utf8(out).expect("summary should be UTF-8") + } + + #[test] + fn summary_reports_the_requested_and_final_urls_with_counts() { + let page = collected( + "https://publisher.example/news/story", + "Example Publisher", + Vec::new(), + ); + + let out = summary(&page, "https://publisher.example/news"); + + assert!( + out.contains("url: https://publisher.example/news\n"), + "should echo the requested URL, got {out:?}" + ); + assert!( + out.contains("final url: https://publisher.example/news/story\n"), + "should report the post-redirect URL, got {out:?}" + ); + assert!(out.contains("scripts: 3"), "got {out:?}"); + assert!(out.contains("resources: 42"), "got {out:?}"); + } + + #[test] + fn page_controlled_text_is_escaped_before_it_reaches_the_terminal() { + // Title, warning text, and the post-redirect URL are all page-controlled. + let page = collected( + "https://publisher.example/a%1B%5B2Jb", + "Example\u{1b}[2J", + vec![Warning { + code: "page_\u{1b}[31m".to_string(), + message: "message\u{1b}[0m".to_string(), + }], + ); + + let out = summary(&page, "https://publisher.example/"); + + assert!( + !out.contains('\u{1b}'), + "no escape sequence may reach the terminal, got {out:?}" + ); + assert!( + out.contains("warning [page_"), + "warnings should still be reported, got {out:?}" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs index 1b48aa1c6..995f21216 100644 --- a/crates/trusted-server-cli/src/commands/config/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -382,6 +382,8 @@ fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), match gate.expected { RuntimeAdStackExpected::Yes => "yes", RuntimeAdStackExpected::No => "no", + // `explain` always supplies a consent decision, which is the only + // input that yields `Unknown`; the arm is here for exhaustiveness. RuntimeAdStackExpected::Unknown => "unknown", } ) @@ -475,11 +477,19 @@ fn format_providers(slot: &CreativeOpportunitySlot) -> String { providers.join(", ") } +/// Renders a set of config-derived slot ids for the terminal. +/// +/// Config can arrive from a pushed blob or the env overlay, not only from a file +/// the operator read, so the ids are escaped before they reach a terminal — the +/// assertion-failure path prints them too. fn join_set(set: &BTreeSet<&str>) -> String { if set.is_empty() { return "(none)".to_string(); } - set.iter().copied().collect::>().join(", ") + set.iter() + .map(|id| escape_terminal_text(id).into_owned()) + .collect::>() + .join(", ") } fn plural(count: usize) -> &'static str { diff --git a/docs/guide/cli.md b/docs/guide/cli.md index df498dfb0..8c7d7e42f 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -229,22 +229,29 @@ survives alongside the homepage's. A wrong ad-unit template makes the publisher bid against inventory that does not exist, so the command prefers a narrow literal path over a plausible guess. -| Situation | Result | -| --------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | -| Only one page was crawled | Literal path. One observation cannot distinguish a literal from a template. | -| The ad unit never varied by section | Literal path. | -| A section's slug is not derivable from its URL (`/site-news` requesting `.../sitenews`) | The slot is omitted and the reason is reported. | -| No root page was seen, so `section_root` is unknown | The slot is omitted rather than writing a guessed fallback. | -| Two path segments could both be the section | No template; the ambiguity is reported. | -| The ad unit varies by device, geo, or anything the URL cannot supply | The refused slot is omitted and the reason is written as a note. | -| Crawled pages report different GAM network ids | The run fails; the pages are not one property. | -| More than a quarter of crawled pages return no slots | The run fails. That is the signature of bot protection serving challenge pages, and writing from it would silently narrow the slot set. | +| Situation | Result | +| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Only one page was crawled | Literal path. One observation cannot distinguish a literal from a template. | +| The ad unit never varied by section | Literal path. | +| A section's slug is not derivable from its URL (`/site-news` requesting `.../sitenews`) | The slot is omitted; the note lists the ad-unit paths it used and says none generalized. | +| No root page was seen, so `section_root` is unknown | The slot is omitted rather than writing a guessed fallback. | +| Two path segments could both be the section | No template; the ambiguity is reported. | +| The ad unit varies by device, geo, or anything the URL cannot supply | The refused slot is omitted and the reason is written as a note. | +| Crawled pages report different GAM network ids | The run fails; the pages are not one property. | +| More than a quarter of crawled pages return no slots | The run fails. That is the signature of bot protection serving challenge pages, and writing from it would silently narrow the slot set. | +| Several live elements normalize onto one div-id prefix | The whole group is omitted, on every page of the crawl. A prefix resolves to at most one element and the exact ids change per render; the prefix is named in a note. | +| A per-render token sits before the placement part of a div id | The slot is omitted from a single observation and the family prefix is named in a note; no stable prefix identifies one element. | Every run checks that the config it produced still loads before replacing the file, and `--dry-run` runs the same check — a clean preview is evidence the config loads, not just that it parses. Dry-run stdout is a zero-context unified diff containing only the managed creative-opportunity fields; notes and refusal -reasons go to stderr, so unrelated config and secrets are not printed. +reasons go to stderr, so unrelated config and secrets are not printed. Crawl +progress also goes to stderr, one line per phase and page — for example +`Auditing desktop [2/17]: /news`. Progress renders the path only, never the +origin, userinfo, query, or fragment, and there is no flag to suppress it. A +`--dry-run` that changes nothing says so on stderr too, leaving stdout an empty +diff. ### Bounding and steering the crawl @@ -268,7 +275,17 @@ hand-tuned fields and gains this run's patterns and newly observed formats, and `gam_unit_path` template is preserved. `--replace` discards existing slots instead, which also discards any template you wrote by hand. -Locale-prefixed sites are inferred at their observed section depth. For +A merge refuses to change the section policy that preserved `{section}` slots +were written against: if the config already sets `section_root` (or +`section_segment`) and this run infers different values, the run fails and asks +for `--replace` as an explicit migration. A config whose `{section}` slots have +no `section_root` at all is a different case — the runtime rejects such a file +outright — so the first merge adopts the inferred policy and makes it loadable +instead of demanding `--replace`. + +Locale-prefixed sites are inferred at their observed section depth. Only real +ISO 639-1 language codes are read as a locale prefix, so a two-letter _section_ +root such as `/tv` or `/us` keeps sections at the first segment. For example, `/en/news/story` can produce `section_segment = 1`; generated patterns retain the locale prefix (`/en/news` and `/en/news/*`). The crawler never invents an unwitnessed locale or section. @@ -407,10 +424,12 @@ ts audit ad-templates verify https://publisher.example/ --allow-cross-origin-red Verification accepts multiple URLs and reuses one browser/profile. Add `--strict` to return exit 1 when a confirmable slot is missing or partially -confirmed, and `--json` for the stable machine-readable report. Video, native, -and out-of-page slots are reported as `unconfirmable`; that records a checker -limitation and does not fail strict mode. `--scroll` enables the optional second -evidence phase and labels evidence first seen after the deterministic scroll. +confirmed, and `--json` for the stable machine-readable report. Video- and +native-only slots are reported as `unconfirmable`; that records a checker +limitation and does not fail strict mode. A live out-of-page slot with no sizes +against banner-configured formats is reported `partial` and does fail strict +mode. `--scroll` enables the optional second evidence phase and labels evidence +first seen after the deterministic scroll. Browser-backed ad-template generation and verification share `--chrome`, `--headful`, `--browser-proxy`, `--no-assume-consent`, diff --git a/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md b/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md index f6dbab2b5..df781e518 100644 --- a/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md +++ b/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md @@ -1056,16 +1056,16 @@ assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "container element id is a valid GPT div match"); } - // §5.4: out-of-page GPT slot is not confirmed; reported as a warning. + // §5.4: out-of-page GPT slot is partial (so it fails strict) plus a warning. #[test] - fn out_of_page_gpt_slot_warns_and_does_not_confirm() { + fn out_of_page_gpt_slot_warns_and_is_partial() { let expected = expected_slot("interstitial", "ad-oop-", "/123/news/oop", &[(300, 250)], &[]); // gpt_slot with empty sizes models an out-of-page slot (no numeric sizes). let evidence = evidence(vec![dom("ad-oop-0")], vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], Vec::new()); let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); - assert_ne!(result.slots[0].status, SlotStatus::Confirmed, "out-of-page is not confirmed in Phase 1"); + assert_eq!(result.slots[0].status, SlotStatus::Partial, "a sizeless slot against banner formats is partial"); assert!(result.slots[0].warnings.iter().any(|w| w.code == "out_of_page_slot")); } @@ -1261,7 +1261,8 @@ - `fluid_size_ignored` — non-numeric observed sizes like `"fluid"` ignored for matching; - `extra_observed_size` — observed GPT sizes not in the configured set; - `configured_size_not_observed` — configured sizes never observed (when ≥1 was); - - `out_of_page_slot` — out-of-page GPT slot observed; not confirmed in Phase 1. + - `out_of_page_slot` — out-of-page GPT slot with no sizes observed; the slot is + reported `partial`, which fails `--strict`. Provider + extra evidence: - APS: configured `providers.aps.slot_id` with matching `fetchBids` → no warning; diff --git a/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md index 3ef89c2ed..873168438 100644 --- a/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md +++ b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md @@ -96,7 +96,8 @@ Cover: - an unrenderable dynamic slot is omitted from expected slots and does not make `matched_slots` pass; - the diagnostic says the runtime omits the slot for that path; - `MediaType` remains typed through comparison; -- video/native-only and out-of-page slots produce `Unconfirmable` and do not fail strict; +- video/native-only slots produce `Unconfirmable` and do not fail strict; +- a sizeless out-of-page slot against banner-configured formats is `Partial` and fails strict; - an incompatible banner is still `Partial` and fails strict; - a missing slot has `phase: None` and JSON omits `phase`; - server-side APS configuration alone does not emit `aps_evidence_missing`; diff --git a/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md index 19a83a801..219cf60c3 100644 --- a/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md +++ b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md @@ -71,8 +71,8 @@ - Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` -- [ ] Add failing registry and request tests for a single `rh-gam-kso__ei_` observation. -- [ ] Add a narrow recognizer requiring the vendor prefix, an eight-or-more-digit mixed alphanumeric token, and a known placement suffix. -- [ ] Omit matching slots while preserving evidence/network discovery and emit one deduplicated actionable diagnostic. -- [ ] Add negative tests proving arbitrary stable IDs sharing only the prefix remain eligible. +- [ ] Add failing registry and request tests for a single `__` observation. +- [ ] Add a recognizer keyed on the token shape — eight or more leading digits followed by more alphanumerics — in any position that still has placement content after it. +- [ ] Omit matching slots while preserving evidence/network discovery and emit one deduplicated actionable diagnostic naming the family prefix. +- [ ] Add negative tests proving IDs with no token, a bare digit run, or a trailing token remain eligible. - [ ] Run the focused tests, then repeat Task 3 verification and delivery. diff --git a/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md index 43229064f..5ff4d0707 100644 --- a/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md +++ b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md @@ -133,9 +133,11 @@ whole config is rejected. Configured media type remains a typed `MediaType` through comparison and is rendered to a string only at the output boundary. Slots that the phase-one -checker cannot confirm (video/native-only and out-of-page) are represented as -unconfirmable and do not fail `--strict`; genuinely partial or missing -confirmable slots still fail. Slot phase is absent when no evidence exists. +checker cannot confirm (video/native-only) are represented as unconfirmable and +do not fail `--strict`; genuinely partial or missing confirmable slots still +fail, including a live out-of-page slot with no sizes matched against +banner-configured formats, which is partial. Slot phase is absent when no +evidence exists. The server-side APS compatibility field no longer creates unconditional client-side `fetchBids` warnings. diff --git a/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md index 143794be8..475751bdb 100644 --- a/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md +++ b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md @@ -12,13 +12,30 @@ preserves the raw IDs, causing `--replace` to write unusable literal slots. ## Design Treat a source-local normalized collision as ambiguous and refuse the entire -group. The first observation remains tentatively accepted. When a second -distinct raw div ID normalizes to the same prefix, remove the first slot, record -the group as ambiguous, and suppress every later member. Emit one diagnostic -when the group first becomes ambiguous, naming the normalized prefix and -explaining that neither a single prefix nor volatile exact IDs are safe. Tell -the operator to expose distinct stable div IDs or prefixes in publisher markup -before configuring the placements. +group. The first observation remains tentatively accepted. When a second raw div +ID that describes a _different element_ normalizes to the same prefix, remove +the first slot, record the group as ambiguous, and suppress every later member. +Emit one diagnostic when the group first becomes ambiguous, naming the +normalized prefix and explaining that neither a single prefix nor volatile exact +IDs are safe. Tell the operator to expose distinct stable div IDs or prefixes in +publisher markup before configuring the placements. + +Two raw IDs sharing a stem are not by themselves two elements. One element +re-rendered under a fresh framework token produces exactly that shape, and +absorbing it is what normalization is for: a React publisher reports +`ad-header-0-_R_3f_` from the server render and `ad-header-0-_r_0_` from the +client one, and refusing that pair would generate no slots at all. The two cases +are separated by comparing what the ephemeral markers did _not_ cover — the +marker spans are excised and the remaining parts compared, so identical +residues mean one element observed twice, while `-in_content-0` against +`-in_content-1` means two siblings and is refused. + +The verdict is site-wide, not page-local. Article pages carry several in-content +units and refuse the shared prefix while a landing page carries one, so a +page-local refusal would let crawl sampling decide whether the ambiguous prefix +reaches the config. `DiscoveredSlots` therefore carries the refused stems, +`EvidenceTable` unions them across pages, and the slot iterator the writer reads +suppresses them regardless of which page contributed them. Registry and request-derived evidence retain separate collision maps, matching the current source precedence: even an ambiguous registry stem continues to @@ -31,35 +48,48 @@ mistaken for a bot challenge. Cross-page slot inference, merging, and `--replace` otherwise remain unchanged because ambiguous slots never enter those stages. -The `rh-gam-kso__ei_` family is independently known -to be volatile across consecutive crawls. Its render token begins with at least -eight digits and continues with mixed alphanumeric entropy. Discovery refuses -even a single otherwise usable registry or request observation of this narrow -family, preserves the page/network evidence, and emits one site-wide diagnostic. -Arbitrary IDs that merely begin with `rh-gam-kso` do not match this rule. +Some ad stacks build IDs as `__`, where the +render token — at least eight leading digits followed by more alphanumerics, +that is, a millisecond timestamp plus entropy — sits _before_ the part that +distinguishes one placement from the next. Such an ID can be written neither +literally nor as a prefix: the only stable prefix stops at the token and reaches +every placement in the family at once. Discovery refuses a single otherwise +usable registry or request observation of that shape, preserves the page/network +evidence, and emits one diagnostic naming the family prefix. The shape decides +rather than a vendor name, so any stack with this layout is covered without a +code change, and every placement after the token is covered rather than an +enumerated few. A token in trailing position is _not_ this case — everything +before it still identifies the element — and is left to normalization and the +collision check. ## Safety and Output The generator prefers omission over a configuration that cannot match future -renders. For the observed Autoblog desktop crawl, replacement output should -therefore contain the stable `ad-header-0` and `ad-fixed_bottom-0` slots, while -the in-content collision group, known `rh-gam-kso` family, and section-varying -sidebar are explained in notes. +renders. For an observed desktop crawl of a site with this mix, replacement +output should therefore contain the stable `ad-header-0` and `ad-fixed_bottom-0` +slots, while the in-content collision group, the volatile-token family, and the +section-varying sidebar are explained in notes. ## Tests - A two-element same-page normalization collision yields no slots and one diagnostic containing the prefix, both unsafe alternatives, and operator action. +- Two renders of one element (identical residues either side of the marker, + including a React server/client pair) collapse to one slot with no diagnostic. - Repeats of the first and second IDs plus a third distinct ID after a collision remain suppressed and do not create additional diagnostics. - Request-derived collisions follow the same policy. - An ambiguous registry stem still suppresses request fallback, and network-ID discovery survives when every collided slot is omitted. +- A stem refused on one page stays refused after a later page contributes a + single member of the group. - A collision-only page is recorded as having evidence rather than as an empty challenge page. -- Single registry- and request-derived `rh-gam-kso` render-token observations - are omitted while retaining evidence and any parseable network ID. -- Stable/nonmatching IDs sharing only the vendor prefix are not omitted. +- Single registry- and request-derived render-token observations are omitted + while retaining evidence and any parseable network ID, for every placement + suffix after the token. +- IDs with no render token, with a bare digit run, or with a trailing token stay + eligible. - Existing normalization, request fallback, fragment detection, and full CLI tests remain green. From 78c0db4539eff66f046376ba084fd1f25d51e3d5 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 16:35:00 +0530 Subject: [PATCH 389/395] Template a slot that never appears on the site root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A placement that only exists on section pages — a sidebar, an in-article unit — witnessed no `section_root` of its own, so inference fell through to a literal decision and refused the slot outright. On a live crawl that dropped `ad-atf_sidebar-0` from the config even though its five observed ad-unit paths differ only in the section segment, and the reported reason ("used several ad-unit paths and none generalized") pointed at the wrong cause. `SlotAnalysis::RootUnwitnessed` now carries the varying segment, so such a slot templates against the config-level `section_root` another slot witnessed. That is safe because the slot's page patterns are derived from the paths it was seen on, all of which carry a section segment: `{section}` never falls back to the root for it. A note names the borrowed `section_root`. When *no* slot witnessed a root, nothing templates, and the diagnostic now says that the crawl never included a page without a section segment instead of blaming generalization. Verified against a live crawl: the sidebar is written with `/{network_id}/autoblog/{section}`, matches only its five sections, and does not match the root, while the previously written slots are unchanged. --- .../commands/audit/generate/unit_template.rs | 138 ++++++++++++++++-- docs/guide/cli.md | 24 +-- 2 files changed, 139 insertions(+), 23 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs index faf8ea67d..ed630e18f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs @@ -96,9 +96,15 @@ enum SlotAnalysis { Static, /// Cannot be represented; carries the operator-facing reason. Refuse(String), - /// Would be templatable but no root page was observed, so `section_root` - /// is undetermined under this candidate. - RootUnwitnessed, + /// Unit segment `varying` tracks the derived section on every page this slot + /// was seen on, but none of those pages lacked the section segment, so the + /// slot witnessed no `section_root` of its own. + /// + /// Carries `varying` because such a slot is still templatable *when another + /// slot witnessed the config-level `section_root`*: a placement that only + /// exists on section pages (a sidebar, an in-article unit) never renders on a + /// path where `{section}` would fall back to the root. + RootUnwitnessed { varying: usize }, } /// Infers unit-path templates for every slot in `table`. @@ -112,6 +118,7 @@ pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> I // Evaluate every candidate index independently; ambiguity between two that // both fit is a refusal, not a preference for the smaller one. let mut qualifying: Vec<(usize, String, BTreeMap)> = Vec::new(); + let mut root_witness_missing = false; for segment in 0..=MAX_SECTION_SEGMENT { let analyses: BTreeMap = slots .iter() @@ -128,6 +135,13 @@ pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> I // Slots must agree: `section_root` is one config-level value, so two // slots claiming different roots means this index is not the real one. let Some(root) = roots.iter().next().copied() else { + // Distinguish "nothing tracks the section" from "everything does but + // no crawled page lacked the section segment": the second is a crawl + // gap the operator can close, and the generic literal-path refusal + // below does not say so. + root_witness_missing |= analyses + .values() + .any(|analysis| matches!(analysis, SlotAnalysis::RootUnwitnessed { .. })); continue; }; if roots.len() > 1 { @@ -155,11 +169,17 @@ pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> I let Some((section_segment, section_root, analyses)) = chosen else { if diagnostics.is_empty() { - diagnostics.push( + diagnostics.push(if root_witness_missing { + "the ad-unit paths do track the page section, but no crawled page lacked a \ + section segment, so `section_root` could not be witnessed and no {section} \ + template can be written; include the site root in the crawl (or set \ + section_root by hand) to template these slots" + .to_string() + } else { "no ad-unit path varied by page section across the crawl, so paths were kept \ literal; crawl more sections to enable a {section} template" - .to_string(), - ); + .to_string() + }); } return InferenceOutcome { policy: None, @@ -175,13 +195,33 @@ pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> I .get(&slot.div_id) .cloned() .unwrap_or(SlotAnalysis::Static); - let decision = match analysis { - SlotAnalysis::Templatable { varying, .. } => { + let templatable = match analysis { + SlotAnalysis::Templatable { varying, .. } => Some((varying, true)), + // The config-level `section_root` is witnessed by another slot on the + // same property, and this slot's page patterns are derived from the + // paths it was seen on — all of which carry a section segment — so + // `{section}` never falls back to the root for it. Refusing here cost + // real inventory: a sidebar or in-article unit that simply does not + // exist on the site root was omitted from the config entirely. + SlotAnalysis::RootUnwitnessed { varying } => Some((varying, false)), + SlotAnalysis::Static | SlotAnalysis::Refuse(_) => None, + }; + let decision = match (templatable, analysis) { + (Some((varying, witnessed_root)), _) => { let template = build_template(slot, varying); match verify_round_trip(&template, slot, network_id, §ion_root, section_segment) { Ok(()) => { templated += 1; + if !witnessed_root { + diagnostics.push(format!( + "slot `{}` was never observed on a page without a section \ + segment, so its `{{section}}` template relies on the \ + config-level section_root `{section_root}` witnessed by other \ + slots; it is only rendered for the paths this slot was seen on", + slot.id + )); + } SlotDecision::Template(template) } Err(reason) => { @@ -194,10 +234,10 @@ pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> I } } } - SlotAnalysis::Static | SlotAnalysis::RootUnwitnessed => literal_decision(slot), - SlotAnalysis::Refuse(reason) => SlotDecision::Refuse { + (None, SlotAnalysis::Refuse(reason)) => SlotDecision::Refuse { reasons: vec![reason], }, + (None, _) => literal_decision(slot), }; decisions.push((slot.div_id.clone(), decision)); } @@ -337,7 +377,7 @@ fn analyse_slot(slot: &SlotEvidence, network_id: &str, section_segment: usize) - let Some(section_root) = roots.next() else { // Without a root observation, `section_root` would be a guess that // silently mis-renders every short path. - return SlotAnalysis::RootUnwitnessed; + return SlotAnalysis::RootUnwitnessed { varying }; }; if roots.next().is_some() { return SlotAnalysis::Static; @@ -659,6 +699,82 @@ mod tests { let SlotDecision::Refuse { .. } = only_decision(&outcome) else { panic!("two literal paths and no template is not representable as one literal"); }; + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("section_root` could not be witnessed")), + "the crawl gap, not \"nothing generalized\", is the reason; got {:?}", + outcome.diagnostics + ); + } + + #[test] + fn a_slot_absent_from_the_root_templates_from_the_witnessed_policy() { + // The live shape behind the `ad-atf_sidebar-0` refusal: a header on the + // root and every section witnesses `section_root`, while a sidebar exists + // only on section pages. The sidebar's unit path tracks the section just + // as well, and its page patterns never cover the root, so refusing it + // dropped real inventory from the config. + let mut table = EvidenceTable::default(); + let pages: &[(&str, &[(&str, &str)])] = &[ + ("/", &[("ad-header", "/123/site/homepage")]), + ( + "/news/story", + &[ + ("ad-header", "/123/site/news"), + ("ad-sidebar", "/123/site/news"), + ], + ), + ( + "/deals/x", + &[ + ("ad-header", "/123/site/deals"), + ("ad-sidebar", "/123/site/deals"), + ], + ), + ]; + for (path, slots) in pages { + let registry: Vec = slots + .iter() + .map(|(div_id, unit_path)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: vec![(728, 90)], + }) + .collect(); + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }), + "the header witnesses the config-level policy" + ); + assert_eq!( + outcome.decision("ad-sidebar"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )), + "a slot that only exists on section pages is still templatable" + ); + assert_eq!( + outcome.decision("ad-header"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )) + ); + assert!( + outcome.diagnostics.iter().any(|note| note + .contains("`ad-sidebar` was never observed on a page without a section segment")), + "the borrowed section_root should be stated; got {:?}", + outcome.diagnostics + ); } #[test] diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 8c7d7e42f..14ca2af31 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -229,18 +229,18 @@ survives alongside the homepage's. A wrong ad-unit template makes the publisher bid against inventory that does not exist, so the command prefers a narrow literal path over a plausible guess. -| Situation | Result | -| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Only one page was crawled | Literal path. One observation cannot distinguish a literal from a template. | -| The ad unit never varied by section | Literal path. | -| A section's slug is not derivable from its URL (`/site-news` requesting `.../sitenews`) | The slot is omitted; the note lists the ad-unit paths it used and says none generalized. | -| No root page was seen, so `section_root` is unknown | The slot is omitted rather than writing a guessed fallback. | -| Two path segments could both be the section | No template; the ambiguity is reported. | -| The ad unit varies by device, geo, or anything the URL cannot supply | The refused slot is omitted and the reason is written as a note. | -| Crawled pages report different GAM network ids | The run fails; the pages are not one property. | -| More than a quarter of crawled pages return no slots | The run fails. That is the signature of bot protection serving challenge pages, and writing from it would silently narrow the slot set. | -| Several live elements normalize onto one div-id prefix | The whole group is omitted, on every page of the crawl. A prefix resolves to at most one element and the exact ids change per render; the prefix is named in a note. | -| A per-render token sits before the placement part of a div id | The slot is omitted from a single observation and the family prefix is named in a note; no stable prefix identifies one element. | +| Situation | Result | +| --------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Only one page was crawled | Literal path. One observation cannot distinguish a literal from a template. | +| The ad unit never varied by section | Literal path. | +| A section's slug is not derivable from its URL (`/site-news` requesting `.../sitenews`) | The slot is omitted; the note lists the ad-unit paths it used and says none generalized. | +| No crawled page lacked a section segment, so `section_root` is unwitnessed | No template is written and the reason names the crawl gap. A slot that merely never appears on the root (a sidebar, an in-article unit) still templates, borrowing the `section_root` another slot witnessed; a note says so. | +| Two path segments could both be the section | No template; the ambiguity is reported. | +| The ad unit varies by device, geo, or anything the URL cannot supply | The refused slot is omitted and the reason is written as a note. | +| Crawled pages report different GAM network ids | The run fails; the pages are not one property. | +| More than a quarter of crawled pages return no slots | The run fails. That is the signature of bot protection serving challenge pages, and writing from it would silently narrow the slot set. | +| Several live elements normalize onto one div-id prefix | The whole group is omitted, on every page of the crawl. A prefix resolves to at most one element and the exact ids change per render; the prefix is named in a note. | +| A per-render token sits before the placement part of a div id | The slot is omitted from a single observation and the family prefix is named in a note; no stable prefix identifies one element. | Every run checks that the config it produced still loads before replacing the file, and `--dry-run` runs the same check — a clean preview is evidence the From 84b84c3c7fe83e1315032f34c94fba5ecf7b7127 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 16:48:45 +0530 Subject: [PATCH 390/395] Scope the retired admin keys reservation to its own namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fallback reservation matched `/admin/keys` on the bare prefix, which also denies unrelated publisher paths that merely start with those bytes — `/admin/keystore`, `/admin/keysets` — while the neighbouring `/admin/keyboards` falls through. The bare prefix is required for `/_ts/admin`, whose unanchored `^/_ts/admin` auth regex authenticates paths like `/_ts/adminec` and would forward their credentials to the origin. No auth handler matches `/admin/keys`, so only the retired alias itself and its separator descendants carry the stale-script risk. Match the alias exactly plus its `/`-separated descendants. Encoded forms stay covered by the percent-decoded evaluation, and the boundary is pinned with `/admin/keystore` alongside the existing `/admin/keyboards` case. --- crates/trusted-server-core/src/ec/admin.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index a94679781..b34a8c0ab 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -96,7 +96,16 @@ fn admin_diagnostic_shape(path: &str) -> Option { /// fallback, because doing so would forward Trusted Server admin credentials /// and request bodies to the publisher origin. fn is_reserved_admin_path(path: &str) -> bool { - path.starts_with(ADMIN_NAMESPACE_PREFIX) || path.starts_with(RETIRED_ADMIN_KEYS_PREFIX) + // `/_ts/admin` is matched on the bare prefix because the unanchored + // `^/_ts/admin` auth regex authenticates those paths too. No auth handler + // matches the retired `/admin/keys` alias, so only the alias itself and its + // separator descendants are reserved — a bare prefix there would also deny + // unrelated publisher paths such as `/admin/keystore`. + path.starts_with(ADMIN_NAMESPACE_PREFIX) + || path == RETIRED_ADMIN_KEYS_PREFIX + || path + .strip_prefix(RETIRED_ADMIN_KEYS_PREFIX) + .is_some_and(|remainder| remainder.starts_with('/')) } /// Percent-decodes `path` once, returning `None` when the path contains no @@ -858,6 +867,7 @@ mod tests { "/admin", "/admin/login", "/admin/keyboards", + "/admin/keystore", "/_ts/api/v1/batch-sync", ] { let request = request_with_method(http::Method::POST, path); From 06b0df8904120cd9ecdab0279a9b1c7ee0e619ab Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 20 Aug 2026 08:55:51 -0500 Subject: [PATCH 391/395] Restore upstream publisher behavior and harden cache policies --- .../src/integrations/registry.rs | 7 - crates/trusted-server-core/src/proxy.rs | 5 + crates/trusted-server-core/src/publisher.rs | 3016 ++++++++++++++--- crates/trusted-server-core/src/settings.rs | 191 +- docs/guide/configuration.md | 28 +- trusted-server.example.toml | 7 +- 6 files changed, 2731 insertions(+), 523 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index ed7970eaf..16cbac868 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1134,13 +1134,6 @@ impl IntegrationRegistry { ids } - /// Return whether an integration is enabled, including integrations whose - /// JavaScript is delivered outside the standard module bundles. - #[must_use] - pub fn is_enabled(&self, integration_id: &str) -> bool { - self.inner.enabled_integration_ids.contains(&integration_id) - } - /// Return JS module IDs for the main (synchronous) bundle, excluding /// modules registered with [`with_deferred_js`](IntegrationRegistrationBuilder::with_deferred_js). #[must_use] diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 2ad8091fd..14485328a 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -115,6 +115,11 @@ pub enum AssetProxyCachePolicy { /// Reapply `Cache-Control: no-store, private` after standard finalization. NoStorePrivate, /// Reapply an operator-selected normalized cache policy after finalization. + /// + /// The adapter must call [`Self::apply_after_route_finalization`] after + /// standard response and privacy finalization, passing its runtime + /// [`EdgeCacheHeader`]. Asset rehosting is Fastly-only today; a future + /// adapter must preserve this finalization step to emit its edge directive. Normalized(CachePolicy), } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 6e94f16f6..a751acaa3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -37,6 +37,7 @@ use http::{HeaderValue, Method, Request, Response, StatusCode, Uri, header}; use crate::auction::endpoints::{ merge_auction_eids, resolve_auction_eids, resolve_client_auction_eids, }; +use crate::auction::formats::sanitize_publisher_page_url; use crate::auction::orchestrator::{ AuctionOrchestrator, DispatchAuctionOutcome, DispatchedAuction, }; @@ -277,13 +278,11 @@ fn accept_encoding_qvalue(header_value: &str, target: &str) -> Option { /// Unified tsjs static serving: `/static/tsjs=` /// -/// Serves three types of bundles: +/// Serves two types of bundles: /// - **Unified bundle** (`tsjs-unified.min.js`): core + immediate (non-deferred) /// integration modules. /// - **Deferred module** (`tsjs-{id}.min.js`): a single self-contained IIFE for -/// modules loaded with `defer` (e.g., Prebid). -/// - **Standalone diagnostics module** (`tsjs-gpt_diagnostics.min.js`): delivered -/// only when the diagnostics integration is enabled and a document activates it. +/// modules loaded with `defer` (e.g., prebid). /// /// # Errors /// @@ -303,19 +302,22 @@ pub fn handle_tsjs_dynamic( let filename = &path[PREFIX.len()..]; if UNIFIED_FILENAMES.contains(&filename) { - // Serve core + immediate modules (excludes deferred like prebid) + // Serve core + immediate modules (excludes deferred like prebid). let module_ids = integration_registry.js_module_ids_immediate(); let body = trusted_server_js::concatenate_modules(&module_ids); let hash = trusted_server_js::concatenated_hash(&module_ids); return Ok(serve_tsjs_static(req, &body, &hash, edge_header)); } - if let Some(module_id) = parse_deferred_module_filename(filename) { + if let Some(module_id) = parse_single_module_filename(filename) { + // Deferred modules and the conditionally injected diagnostics module + // are served as content-addressed standalone assets. Delivery remains + // cookie-independent so the static response can stay publicly cached. let deferred_ids = integration_registry.js_module_ids_deferred(); - let is_enabled_diagnostics_module = module_id + let diagnostics_standalone = module_id == crate::integrations::gpt_diagnostics::GPT_DIAGNOSTICS_INTEGRATION_ID - && integration_registry.is_enabled(module_id); - if !deferred_ids.contains(&module_id) && !is_enabled_diagnostics_module { + && integration_registry.integration_enabled(module_id); + if !deferred_ids.contains(&module_id) && !diagnostics_standalone { return Ok(not_found_response()); } if let (Some(content), Some(hash)) = ( @@ -335,7 +337,7 @@ fn serve_tsjs_static( expected_hash: &str, edge_header: EdgeCacheHeader, ) -> Response { - let mut resp = serve_static_with_etag( + let mut response = serve_static_with_etag( body, req, "application/javascript; charset=utf-8", @@ -343,11 +345,12 @@ fn serve_tsjs_static( ); if request_version_hash(req).is_some_and(|hash| hash == expected_hash) { CachePolicy::public_immutable(Duration::from_secs(31_536_000)) - .apply_to_headers(resp.headers_mut(), edge_header); + .apply_to_headers(response.headers_mut(), edge_header); } - resp.headers_mut() + response + .headers_mut() .insert(HEADER_X_COMPRESS_HINT, HeaderValue::from_static("on")); - resp + response } fn request_version_hash(req: &Request) -> Option<&str> { @@ -357,13 +360,13 @@ fn request_version_hash(req: &Request) -> Option<&str> { }) } -/// Extract a module ID from a deferred-module filename like `tsjs-prebid.min.js`. +/// Extract a module ID from a deferred-module filename like `tsjs-sourcepoint.min.js`. /// /// Returns `Some(&'static str)` if the filename matches a known JS module ID, /// `None` otherwise. The caller must additionally verify that the module is /// both deferred and enabled via the [`IntegrationRegistry`]. #[must_use] -fn parse_deferred_module_filename(filename: &str) -> Option<&'static str> { +fn parse_single_module_filename(filename: &str) -> Option<&'static str> { let stem = filename .strip_prefix("tsjs-") .and_then(|s| s.strip_suffix(".min.js").or_else(|| s.strip_suffix(".js")))?; @@ -1105,7 +1108,14 @@ fn apply_publisher_asset_cache_policy( response: &mut Response, ) -> Result<(), Report> { let is_cacheable_method = *method == Method::GET || *method == Method::HEAD; - if !is_cacheable_method || response_cache_control_is_private_or_no_store(response) { + if !is_cacheable_method + || response_cache_control_is_private_or_no_store(response) + || response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(is_html_content_type) + { return Ok(()); } @@ -1503,21 +1513,6 @@ fn strip_conditional_and_range_headers(req: &mut Request) { req.headers_mut().remove(header::IF_RANGE); } -/// Prevent shared caches from replaying tag-suppressed HTML to other clients. -fn apply_datadome_client_tag_cache_privacy( - response: &mut Response, - method: &Method, - suppress_datadome_client_side_tag: bool, - content_type: &str, -) { - if suppress_datadome_client_side_tag - && response_carries_body(method, response.status()) - && is_html_content_type(content_type) - { - enforce_synthesized_html_cache_privacy(response); - } -} - /// Returns `true` when a buffered publisher response should carry a body and a /// recomputed `Content-Length`. /// @@ -1533,6 +1528,21 @@ fn response_carries_body(method: &Method, status: StatusCode) -> bool { && status != StatusCode::NOT_MODIFIED } +/// Prevent shared caches from replaying tag-suppressed HTML to other clients. +fn apply_datadome_client_tag_cache_privacy( + response: &mut Response, + method: &Method, + suppress_datadome_client_side_tag: bool, + content_type: &str, +) { + if suppress_datadome_client_side_tag + && response_carries_body(method, response.status()) + && is_html_content_type(content_type) + { + enforce_synthesized_html_cache_privacy(response); + } +} + /// Drop a bodiless response's body and correct its framing headers. /// /// The response keeps no body, and its `Content-Length` is corrected where the @@ -1898,21 +1908,25 @@ pub(crate) fn write_bids_to_state( settings: &Settings, request_origin: &str, include_debug_bid: bool, -) { + auction_id: Option<&str>, +) -> std::collections::HashSet { log::debug!( "write_bids_to_state: {} winning bid(s): [{}]", winning_bids.len(), winning_bids.keys().cloned().collect::>().join(", ") ); - let bid_map = build_bid_map( + let bid_map = build_bid_map_with_auction_id( winning_bids, price_granularity, settings, request_origin, include_debug_bid, + auction_id, ); + let delivered_winner_slots = bid_map.keys().cloned().collect(); let bids_script = build_bids_script(&bid_map); *ad_bids_state.lock().expect("should lock bid state") = Some(bids_script); + delivered_winner_slots } /// Maximum serialized size (in bytes) of a dump embedded in the `ts-debug` @@ -2497,6 +2511,10 @@ async fn collect_non_html_auction( services: &RuntimeServices, settings: &Settings, ) { + let auction_id = telemetry + .auction_request + .as_ref() + .and_then(|_| diagnostics_auction_id(settings)); let placeholder = mediator_placeholder_request(); let result = orchestrator .collect_dispatched_auction( @@ -2505,6 +2523,15 @@ async fn collect_non_html_auction( &make_collect_context(settings, services, &placeholder), ) .await; + let delivered_winner_slots = write_bids_to_state( + &result.winning_bids, + params.price_granularity, + ¶ms.ad_bids_state, + settings, + &request_origin(¶ms.request_scheme, ¶ms.request_host), + settings.debug.inject_adm_for_testing, + auction_id.as_deref(), + ); if let (Some(observation), Some(auction_request)) = (telemetry.observation, telemetry.auction_request.as_ref()) { @@ -2514,20 +2541,12 @@ async fn collect_non_html_auction( AuctionTerminalOutcome::Completed { request: auction_request, result: &result, - delivered_winner_slots: None, + delivered_winner_slots: Some(&delivered_winner_slots), }, ) }) .await; } - write_bids_to_state( - &result.winning_bids, - params.price_granularity, - ¶ms.ad_bids_state, - settings, - &request_origin(¶ms.request_scheme, ¶ms.request_host), - settings.debug.inject_adm_for_testing, - ); } // Private orchestration helper called only from `body_close_hold_loop`. @@ -2546,12 +2565,29 @@ async fn collect_stream_auction( settings, request_origin, } = deps; + let auction_id = telemetry + .auction_request + .as_ref() + .and_then(|_| diagnostics_auction_id(settings)); log::info!("body_close_hold_loop: collecting dispatched auction before held body tail"); let placeholder = mediator_placeholder_request(); let collect_ctx = make_collect_context(settings, services, &placeholder); let result = orchestrator .collect_dispatched_auction(dispatched, services, &collect_ctx) .await; + log::info!( + "body_close_hold_loop: collect complete - {} winning bid(s)", + result.winning_bids.len() + ); + let delivered_winner_slots = write_bids_to_state( + &result.winning_bids, + *price_granularity, + ad_bids_state, + settings, + request_origin, + settings.debug.inject_adm_for_testing, + auction_id.as_deref(), + ); if let (Some(observation), Some(auction_request)) = (telemetry.observation, telemetry.auction_request.as_ref()) { @@ -2561,24 +2597,12 @@ async fn collect_stream_auction( AuctionTerminalOutcome::Completed { request: auction_request, result: &result, - delivered_winner_slots: None, + delivered_winner_slots: Some(&delivered_winner_slots), }, ) }) .await; } - log::info!( - "body_close_hold_loop: collect complete - {} winning bid(s)", - result.winning_bids.len() - ); - write_bids_to_state( - &result.winning_bids, - *price_granularity, - ad_bids_state, - settings, - request_origin, - settings.debug.inject_adm_for_testing, - ); if settings.debug.auction_html_comment { prepend_auction_debug_comment("stream", &result, ad_bids_state); @@ -2734,8 +2758,7 @@ pub async fn handle_publisher_request( log::debug!("Proxying request to configured publisher backend"); let request_path = req.uri().path().to_string(); - let request_method = req.method().clone(); - let is_get = request_method == Method::GET; + let is_get = req.method() == http::Method::GET; let is_prefetch = is_prefetch_request(&req); let is_bot = is_bot_user_agent(&req); @@ -2972,6 +2995,7 @@ pub async fn handle_publisher_request( // sets the flag unconditionally and tolerates buffered fallback): adapters // without streaming support may reject the flag outright rather than // silently buffering, which would fail every publisher fetch. + let request_method = req.method().clone(); let mut platform_request = PlatformHttpRequest::new(req, backend_name); if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); @@ -3041,7 +3065,7 @@ pub async fn handle_publisher_request( // §4.7: HTML with synthesized per-navigation auction state must not be // stored or validated as an origin representation. Strip both browser and - // edge-cache validators/directives before returning it. + // surrogate validators/cache directives before returning it. // // Gate on `should_run_ad_stack` rather than content-type alone: when no slot // matched, the feature is disabled, or this is not an ad-eligible navigation, @@ -3053,7 +3077,7 @@ pub async fn handle_publisher_request( let origin_content_type = response .headers() .get(header::CONTENT_TYPE) - .and_then(|h| h.to_str().ok()) + .and_then(|value| value.to_str().ok()) .unwrap_or_default() .to_string(); if should_run_ad_stack && is_html_content_type(&origin_content_type) { @@ -3065,7 +3089,6 @@ pub async fn handle_publisher_request( suppress_datadome_client_side_tag, &origin_content_type, ); - apply_publisher_asset_cache_policy( settings, &request_path, @@ -3178,11 +3201,11 @@ pub async fn handle_publisher_request( content_type, ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), + suppress_datadome_client_side_tag, auction_observation, auction_request: auction_request_for_telemetry, dispatched_auction, price_granularity, - suppress_datadome_client_side_tag, gpt_diagnostics: Some(gpt_diagnostics), }), }) @@ -3282,10 +3305,11 @@ pub(crate) fn build_auction_request( // so SSPs, injected creatives, and brand-safety pixels see the publisher's // own origin. On the SSAT proxy path `request_info.host` is the trusted // server edge host, which must not leak into the bid request. - let page_url = format!( + let page_candidate = format!( "{}://{}{}", request_info.scheme, publisher_domain, slots_ctx.request_path ); + let page_url = sanitize_publisher_page_url(Some(&page_candidate), publisher_domain); let ec_id = ec_id.filter(|id| !id.is_empty()); let request_id = ec_id.map_or_else( || format!("ts-req-{}", uuid::Uuid::new_v4().simple()), @@ -3316,6 +3340,21 @@ pub(crate) fn build_auction_request( } } +/// Mint the browser-visible auction correlation token for GPT diagnostics. +/// +/// The token is freshly generated per auction and carries no user identity. +/// [`AuctionRequest::id`] must never be used here: for a consented visitor it is +/// `ts-{ec_id}`, so publishing it in `window.tsjs.bids` would hand the `HttpOnly` +/// EC identifier to any script on the page, and — being stable per visitor — it +/// could not distinguish one auction from the next either. +/// +/// Returns `None` unless the GPT diagnostics integration is enabled, since +/// nothing else consumes the value. +fn diagnostics_auction_id(settings: &Settings) -> Option { + crate::integrations::gpt_diagnostics::is_enabled(settings) + .then(|| format!("ts-auc-{}", uuid::Uuid::new_v4().simple())) +} + /// Escape a JSON string so it is safe to embed inside a JS double-quoted string literal /// inside an HTML `", + "", escaped ) } @@ -3521,8 +3709,9 @@ pub(crate) fn build_empty_bids_script() -> String { /// [`handle_page_bids`] (SPA navigation) so the slot wire shape has a single /// definition and the two paths cannot silently diverge. Property names match /// what the client-side TSJS bundle expects: `gam_unit_path`, `div_id`, -/// `formats`, and `targeting`. -fn build_slot_json( +/// `formats`, and `targeting`. Returns `None` when the slot's dynamic GAM unit +/// path exceeds its rendering limit. +pub(crate) fn build_slot_json( slot: &crate::creative_opportunities::CreativeOpportunitySlot, co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, section: &str, @@ -3584,6 +3773,8 @@ pub(crate) fn build_ad_slots_script( co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, request_path: &str, ) -> String { + // `{section}` derives from the same raw path `page_patterns` matched + // against; derive it once for every slot on this request. let section = co_config.section_for_path(request_path); let slots: Vec = matched_slots .iter() @@ -3872,8 +4063,8 @@ pub async fn handle_page_bids( // skip the live auction, matching the existing bot/prefetch behaviour. let ad_stack_enabled = auction_enabled && consent_allows_auction; - let winning_bids = if matched_slots.is_empty() { - std::collections::HashMap::new() + let (winning_bids, prebuilt_bid_map) = if matched_slots.is_empty() { + (std::collections::HashMap::new(), None) } else { // Same publisher identity as the outbound bid request — see the // matching note on the initial-navigation observation above. @@ -3929,18 +4120,28 @@ pub async fn handle_page_bids( { Ok(result) => { let winning_bids = result.winning_bids.clone(); + let auction_id = diagnostics_auction_id(settings); + let bid_map = build_bid_map_with_auction_id( + &winning_bids, + co_config.price_granularity, + settings, + &page_bids_request_origin, + settings.debug.inject_adm_for_testing, + auction_id.as_deref(), + ); + let delivered_winner_slots = bid_map.keys().cloned().collect(); emit_auction_events_best_effort_lazy(services, || { build_auction_events( observation, AuctionTerminalOutcome::Completed { request: &auction_request, result: &result, - delivered_winner_slots: None, + delivered_winner_slots: Some(&delivered_winner_slots), }, ) }) .await; - winning_bids + (winning_bids, Some(bid_map)) } Err(e) => { log::warn!("page-bids auction failed: {e:?}"); @@ -3957,7 +4158,7 @@ pub async fn handle_page_bids( ) }) .await; - std::collections::HashMap::new() + (std::collections::HashMap::new(), None) } } } else { @@ -3983,17 +4184,20 @@ pub async fn handle_page_bids( ) }) .await; - std::collections::HashMap::new() + (std::collections::HashMap::new(), None) } }; - let bid_map = build_bid_map( - &winning_bids, - co_config.price_granularity, - settings, - &page_bids_request_origin, - settings.debug.inject_adm_for_testing, - ); + let bid_map = prebuilt_bid_map.unwrap_or_else(|| { + build_bid_map_with_auction_id( + &winning_bids, + co_config.price_granularity, + settings, + &page_bids_request_origin, + settings.debug.inject_adm_for_testing, + None, + ) + }); // Gate slots on the ad-stack kill switch / consent: when disabled, return no // slots so the SPA hook does not call `adInit()` / create GPT slots. @@ -4071,7 +4275,8 @@ mod tests { use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::integrations::IntegrationRegistry; use crate::platform::test_support::{ - StubHttpClient, build_services_with_http_client, noop_services, + NoopSecretStore, StubHttpClient, build_services_with_http_client, + build_services_with_secret_http_client_and_client_ip, noop_services, noop_services_with_telemetry_sink, }; use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; @@ -4358,8 +4563,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: Default::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -4566,68 +4771,7 @@ mod tests { } #[tokio::test] - async fn publisher_request_uses_platform_http_client_with_http_types() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"origin response".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let response = match run_publisher_proxy(&settings, &services, req).await { - PublisherResponse::Buffered(r) => r, - PublisherResponse::PassThrough { mut response, body } => { - *response.body_mut() = body; - response - } - PublisherResponse::Stream { response, .. } => response, - }; - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response_body_string(response), "origin response"); - assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "should proxy through the platform http client" - ); - } - - #[tokio::test] - async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let _ = run_publisher_proxy(&settings, &services, req).await; - - assert_eq!( - stub.recorded_stream_response_flags(), - vec![false], - "publisher origin fetch must not request streams when the platform does not support them" - ); - } - - #[tokio::test] - async fn publisher_request_applies_configured_asset_cache_policy() { + async fn publisher_asset_cache_policy_applies_to_non_html_response() { let settings = Settings::from_toml(&format!( r#"{} @@ -4656,18 +4800,16 @@ mod tests { let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); - let req = HttpRequest::builder() + let request = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/assets/logo.0123abcd.png") .header(header::HOST, "publisher.example") .body(EdgeBody::empty()) .expect("should build request"); - let response = match run_publisher_proxy(&settings, &services, req).await { - PublisherResponse::PassThrough { response, .. } => response, - PublisherResponse::Buffered(response) | PublisherResponse::Stream { response, .. } => { - response - } + let response = run_publisher_proxy(&settings, &services, request).await; + let PublisherResponse::PassThrough { response, .. } = response else { + panic!("should pass through non-HTML asset response"); }; assert_eq!( @@ -4676,7 +4818,7 @@ mod tests { .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), Some("public, max-age=31536000, immutable"), - "matched publisher-origin asset should receive normalized immutable policy" + "matched publisher asset should receive immutable browser policy" ); assert_eq!( response @@ -4684,53 +4826,85 @@ mod tests { .get("surrogate-control") .and_then(|value| value.to_str().ok()), Some("max-age=31536000"), - "publisher-origin asset should receive selected runtime edge header" + "matched publisher asset should receive Fastly edge policy" ); } #[tokio::test] - async fn publisher_origin_fetch_sets_stream_response_when_supported() { - let settings = create_test_settings(); + async fn publisher_asset_policy_response_with_cookie_is_private_after_finalization() { + let settings = Settings::from_toml(&format!( + r#"{} + + [[cache.asset_rules]] + id = "publisher-fingerprinted-assets" + enabled = true + path_globs = ["/assets/**/*.png"] + fingerprint_style = "hex" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + )) + .expect("should parse settings with cache rule"); let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); stub.push_response_with_headers( 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], + b"png".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "image/png"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + (header::SET_COOKIE.as_str(), "viewer=example; Path=/"), + ], ); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); - let req = HttpRequest::builder() + let request = HttpRequest::builder() .method(Method::GET) - .uri("https://publisher.example/page") + .uri("https://publisher.example/assets/logo.0123abcd.png") .header(header::HOST, "publisher.example") .body(EdgeBody::empty()) .expect("should build request"); - let _ = run_publisher_proxy(&settings, &services, req).await; + let response = run_publisher_proxy(&settings, &services, request).await; + let PublisherResponse::PassThrough { mut response, .. } = response else { + panic!("should pass through non-HTML asset response"); + }; + crate::response_privacy::apply_response_headers_with_cache_privacy( + &settings, + &mut response, + ); assert_eq!( - stub.recorded_stream_response_flags(), - vec![true], - "publisher origin fetch should request streams when the platform supports them" + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "publisher asset with Set-Cookie must become private after finalization" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "publisher asset with Set-Cookie must not retain a shared-cache header" ); } #[tokio::test] - async fn publisher_asset_cache_policy_respects_split_no_store_origin_header() { + async fn publisher_asset_cache_policy_skips_html_response() { let settings = Settings::from_toml(&format!( r#"{} [[cache.asset_rules]] - id = "publisher-fingerprinted-assets" + id = "broad-publisher-path" enabled = true - path_globs = ["/assets/**/*.png"] - fingerprint_style = "hex" + path_glob = "/news/*.html" visibility = "public" browser_ttl_seconds = 31536000 edge_ttl_seconds = 31536000 immutable = true + fingerprint_style = "hex" "#, crate_test_settings_str() )) @@ -4738,44 +4912,921 @@ mod tests { let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( 200, - b"png".to_vec(), + b"news".to_vec(), vec![ - (header::CONTENT_TYPE.as_str(), "image/png"), + (header::CONTENT_TYPE.as_str(), "text/html; charset=utf-8"), (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - (header::CACHE_CONTROL.as_str(), "no-store"), ], ); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); - let req = HttpRequest::builder() + let request = HttpRequest::builder() .method(Method::GET) - .uri("https://publisher.example/assets/logo.0123abcd.png") + .uri("https://publisher.example/news/story.0123abcd.html") .header(header::HOST, "publisher.example") .body(EdgeBody::empty()) .expect("should build request"); - let response = match run_publisher_proxy(&settings, &services, req).await { - PublisherResponse::PassThrough { response, .. } => response, - PublisherResponse::Buffered(response) | PublisherResponse::Stream { response, .. } => { + let response = run_publisher_proxy(&settings, &services, request).await; + let response = match response { + PublisherResponse::Stream { response, .. } | PublisherResponse::Buffered(response) => { response } + PublisherResponse::PassThrough { .. } => { + panic!("should classify HTML response for processing") + } }; - let cache_control_values = response - .headers() - .get_all(header::CACHE_CONTROL) - .iter() - .filter_map(|value| value.to_str().ok()) - .collect::>(); assert_eq!( - cache_control_values, - vec!["public, max-age=60", "no-store"], - "origin no-store in a later Cache-Control field should prevent normalized upgrade" - ); + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=60"), + "asset policy must not cache publisher HTML" + ); assert!( response.headers().get("surrogate-control").is_none(), - "origin no-store response must not receive edge-cache headers" + "HTML response must not receive a shared-cache header" + ); + } + + mod ssat_cache_policy_tests { + use super::*; + use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; + use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; + use crate::platform::test_support::{ + NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, + }; + use crate::platform::{ + ClientInfo, PlatformError, PlatformHttpClient, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, + }; + use crate::test_support::tests::crate_test_settings_str; + + const ORIGIN_ETAG: &str = "\"origin-tag\""; + const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; + const UNEXPECTED_304_PROVIDER: &str = "example_navigation_bidder"; + const UNEXPECTED_304_BACKEND: &str = "example-navigation-bidder-backend"; + + struct DispatchingTestProvider; + + struct RangeAwareHttpClient { + stub: StubHttpClient, + } + + impl RangeAwareHttpClient { + fn new() -> Self { + Self { + stub: StubHttpClient::new(), + } + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for RangeAwareHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + if request.request.headers().contains_key(header::RANGE) { + self.stub.push_response_with_headers( + 206, + b"partial".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("content-range", "bytes 0-18/39"), + ], + ); + } else { + self.stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + } + self.stub.send(request).await + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.stub.send_async(request).await + } + + async fn select( + &self, + pending_requests: Vec, + ) -> Result> { + self.stub.select(pending_requests).await + } + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for DispatchingTestProvider { + fn provider_name(&self) -> &'static str { + UNEXPECTED_304_PROVIDER + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let request = PlatformHttpRequest::new( + HttpRequest::builder() + .method(Method::POST) + .uri("https://bidder.example.com/navigation-bids") + .body(EdgeBody::empty()) + .expect("should build test provider request"), + UNEXPECTED_304_BACKEND, + ); + context + .services + .http_client() + .send_async(request) + .await + .change_context(TrustedServerError::Auction { + message: "test provider launch failed".to_string(), + }) + .map(ProviderRequestOutcome::pending) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("parse_response must not run for an unexpected origin 304"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name( + &self, + _services: &RuntimeServices, + _timeout_ms: u32, + ) -> Option { + Some(UNEXPECTED_304_BACKEND.to_string()) + } + } + + #[derive(Default)] + struct RecordingTelemetrySink { + batches: Mutex>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionTelemetrySink for RecordingTelemetrySink { + async fn emit_auction_events( + &self, + _services: &RuntimeServices, + batch: AuctionEventBatch, + ) -> Result<(), Report> { + self.batches + .lock() + .expect("should lock telemetry batches") + .push(batch); + Ok(()) + } + } + + fn settings_with_enabled_auction_and_creative_opportunities() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n\ + [creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml) + .expect("should parse settings with auction and creative opportunities enabled") + } + + fn settings_with_dispatching_provider() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\nproviders = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n\ + [creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml) + .expect("should parse settings with the dispatching test provider") + } + + fn services_with_telemetry( + http_client: Arc, + telemetry_sink: Arc, + ) -> RuntimeServices { + let telemetry_sink: Arc = telemetry_sink; + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(http_client) + .geo(Arc::new(NoopGeo)) + .auction_telemetry_sink(telemetry_sink) + .client_info(ClientInfo::default()) + .build() + } + + fn article_slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: "article-slot".to_string(), + gam_unit_path: None, + div_id: None, + page_patterns: vec!["/article".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }], + floor_price: None, + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + compiled_unit: None, + } + } + + fn conditional_navigation_request() -> Request { + HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::IF_NONE_MATCH, ORIGIN_ETAG) + .header(header::IF_MODIFIED_SINCE, ORIGIN_LAST_MODIFIED) + .body(EdgeBody::empty()) + .expect("should build conditional navigation request") + } + + fn queue_cacheable_html_response(stub: &StubHttpClient) { + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ("cdn-cache-control", "max-age=300"), + ("cloudflare-cdn-cache-control", "max-age=300"), + ], + ); + } + + async fn run_with_slots( + settings: &Settings, + services: &RuntimeServices, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> PublisherResponse { + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + run_with_orchestrator(settings, services, &orchestrator, slots, req).await + } + + async fn run_with_orchestrator( + settings: &Settings, + services: &RuntimeServices, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> PublisherResponse { + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + + handle_publisher_request( + settings, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator, + slots, + registry: None, + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request") + } + + fn response_head(response: PublisherResponse) -> http::response::Parts { + match response { + PublisherResponse::Buffered(response) + | PublisherResponse::Stream { response, .. } + | PublisherResponse::PassThrough { response, .. } => response.into_parts().0, + } + } + + fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } + + #[tokio::test] + async fn eligible_navigation_bypasses_cache_and_returns_non_storable_html() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + let req = conditional_navigation_request(); + + // Act + let response = run_with_slots(&settings, &services, &slots, req).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![true], + "eligible publisher navigation should bypass the platform cache" + ); + let recorded_requests = stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + assert_eq!( + recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), + None, + "eligible publisher request should not forward If-None-Match" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), + None, + "eligible publisher request should not forward If-Modified-Since" + ); + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "eligible HTML response should be private and non-storable" + ); + for header_name in [ + header::ETAG, + header::LAST_MODIFIED, + header::HeaderName::from_static("surrogate-control"), + header::HeaderName::from_static("fastly-surrogate-control"), + header::HeaderName::from_static("cdn-cache-control"), + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + ] { + assert!( + !response_head.headers.contains_key(&header_name), + "eligible HTML response should remove {header_name}" + ); + } + } + + #[tokio::test] + async fn eligible_range_navigation_fetches_complete_html() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let http_client = Arc::new(RangeAwareHttpClient::new()); + let services = build_services_with_http_client( + Arc::clone(&http_client) as Arc + ); + let slots = [article_slot()]; + let mut req = conditional_navigation_request(); + req.headers_mut() + .insert(header::RANGE, HeaderValue::from_static("bytes=0-18")); + req.headers_mut() + .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); + + // Act + let response = run_with_slots(&settings, &services, &slots, req).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head.status, + StatusCode::OK, + "eligible range navigation should fetch the complete origin document" + ); + let recorded_requests = http_client.stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + for header_name in [header::RANGE, header::IF_RANGE] { + assert_eq!( + recorded_header(outbound_headers, header_name.as_str()), + None, + "eligible publisher request should not forward {header_name}" + ); + } + } + + #[tokio::test] + async fn navigation_without_matched_slots_preserves_origin_cache_policy() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = conditional_navigation_request(); + req.headers_mut() + .insert(header::RANGE, HeaderValue::from_static("bytes=0-18")); + req.headers_mut() + .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); + + // Act + let response = run_with_slots(&settings, &services, &[], req).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "publisher navigation without matched slots should use the default cache mode" + ); + let recorded_requests = stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + assert_eq!( + recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), + Some(ORIGIN_ETAG), + "publisher request without matched slots should preserve If-None-Match" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), + Some(ORIGIN_LAST_MODIFIED), + "publisher request without matched slots should preserve If-Modified-Since" + ); + assert_eq!( + recorded_header(outbound_headers, header::RANGE.as_str()), + Some("bytes=0-18"), + "publisher request without matched slots should preserve Range" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_RANGE.as_str()), + Some(ORIGIN_ETAG), + "publisher request without matched slots should preserve If-Range" + ); + + for (header_name, expected) in [ + (header::CACHE_CONTROL, "public, max-age=300"), + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "publisher response without matched slots should preserve {header_name}" + ); + } + } + + #[tokio::test] + async fn eligible_navigation_rejects_unexpected_origin_304() { + for content_type in [None, Some("text/html; charset=utf-8")] { + // Arrange + let settings = settings_with_dispatching_provider(); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let stub = Arc::new(StubHttpClient::new()); + + // `send_async` consumes the first response before the publisher + // origin request consumes the second response. + stub.push_response(200, b"unused provider response".to_vec()); + let mut origin_headers = vec![ + ("cache-control", "public, max-age=300"), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ]; + if let Some(content_type) = content_type { + origin_headers.push(("content-type", content_type)); + } + stub.push_response_with_headers(304, Vec::new(), origin_headers); + let services = services_with_telemetry( + Arc::clone(&stub) as Arc, + Arc::clone(&telemetry_sink), + ); + let slots = [article_slot()]; + + // Act + let response = run_with_orchestrator( + &settings, + &services, + &orchestrator, + &slots, + conditional_navigation_request(), + ) + .await; + + // Assert + let response = match response { + PublisherResponse::Buffered(response) => response, + PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + panic!("unexpected origin 304 should return a buffered response") + } + }; + assert_eq!( + response.status(), + StatusCode::BAD_GATEWAY, + "eligible origin 304 should fail closed with or without Content-Type" + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, no-store"), + "eligible origin 304 should return an explicitly non-storable response" + ); + for header_name in [ + header::ETAG, + header::LAST_MODIFIED, + header::HeaderName::from_static("surrogate-control"), + header::HeaderName::from_static("fastly-surrogate-control"), + ] { + assert!( + !response.headers().contains_key(&header_name), + "eligible origin 304 should not forward {header_name}" + ); + } + + let batches = telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches"); + let summary_rows: Vec<_> = batches + .iter() + .flat_map(AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summary_rows.len(), + 1, + "unexpected origin 304 should emit exactly one summary row" + ); + assert_eq!( + summary_rows[0].terminal_status.as_deref(), + Some("abandoned"), + "unexpected origin 304 should abandon the dispatched auction" + ); + assert_eq!( + summary_rows[0].terminal_reason.as_deref(), + Some("unexpected_origin_304"), + "unexpected origin 304 should use the bounded telemetry reason" + ); + } + } + + #[tokio::test] + async fn noneligible_origin_304_preserves_conditional_response_metadata() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 304, + Vec::new(), + vec![ + ("cache-control", "public, max-age=300"), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; + + // Assert + let response = match response { + PublisherResponse::Buffered(response) => response, + PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + panic!("noneligible origin 304 should remain buffered") + } + }; + assert_eq!( + response.status(), + StatusCode::NOT_MODIFIED, + "noneligible origin 304 should preserve its status" + ); + for (header_name, expected) in [ + (header::CACHE_CONTROL, "public, max-age=300"), + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ] { + assert_eq!( + response + .headers() + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "noneligible origin 304 should preserve {header_name}" + ); + } + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "noneligible publisher navigation should use the default cache mode" + ); + let recorded_requests = stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + assert_eq!( + recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), + Some(ORIGIN_ETAG), + "noneligible publisher request should preserve If-None-Match" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), + Some(ORIGIN_LAST_MODIFIED), + "noneligible publisher request should preserve If-Modified-Since" + ); + } + } + + #[tokio::test] + async fn publisher_request_uses_platform_http_client_with_http_types() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"origin response".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = match run_publisher_proxy(&settings, &services, req).await { + PublisherResponse::Buffered(r) => r, + PublisherResponse::PassThrough { mut response, body } => { + *response.body_mut() = body; + response + } + PublisherResponse::Stream { response, .. } => response, + }; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body_string(response), "origin response"); + assert_eq!( + stub.recorded_backend_names(), + vec!["stub-backend".to_string()], + "should proxy through the platform http client" + ); + } + + #[tokio::test] + async fn suppressed_navigation_removes_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .header(header::IF_NONE_MATCH, "\"cached-page\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-page\"") + .body(EdgeBody::empty()) + .expect("should build conditional request"); + req.extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for header_name in [ + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::RANGE, + header::IF_RANGE, + ] { + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), + "suppressed navigations must not forward {header_name}" + ); + } + } + + #[tokio::test] + async fn suppressed_iframe_removes_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"frame".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/frame") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "iframe") + .header(header::IF_NONE_MATCH, "\"cached-frame\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-frame\"") + .body(EdgeBody::empty()) + .expect("should build conditional iframe request"); + req.extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for header_name in [ + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::RANGE, + header::IF_RANGE, + ] { + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), + "suppressed iframe documents must not forward {header_name}" + ); + } + } + + #[tokio::test] + async fn suppressed_subresource_preserves_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"video".to_vec(), + vec![("content-type", "video/mp4")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/video.mp4") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "video") + .header(header::IF_NONE_MATCH, "\"cached-video\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-video\"") + .body(EdgeBody::empty()) + .expect("should build conditional subresource request"); + req.extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for (header_name, expected) in [ + (header::IF_NONE_MATCH, "\"cached-video\""), + (header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT"), + (header::RANGE, "bytes=0-18"), + (header::IF_RANGE, "\"cached-video\""), + ] { + assert_eq!( + headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(header_name.as_str())) + .map(|(_, value)| value.as_str()), + Some(expected), + "suppressed subresources should preserve {header_name}" + ); + } + } + + #[tokio::test] + async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let _ = run_publisher_proxy(&settings, &services, req).await; + + assert_eq!( + stub.recorded_stream_response_flags(), + vec![false], + "publisher origin fetch must not request streams when the platform does not support them" + ); + } + + #[tokio::test] + async fn publisher_origin_fetch_sets_stream_response_when_supported() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.set_streaming_responses_supported(true); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let _ = run_publisher_proxy(&settings, &services, req).await; + + assert_eq!( + stub.recorded_stream_response_flags(), + vec![true], + "publisher origin fetch should request streams when the platform supports them" ); } @@ -4826,15 +5877,126 @@ mod tests { registry: None, }, req, - EdgeCacheHeader::SurrogateControl, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + + assert_eq!( + ec_context.ec_value(), + None, + "handler must not self-generate an EC ID; generation is the adapter's real-browser-gated responsibility", + ); + } + + #[tokio::test] + async fn datadome_filter_marker_survives_into_publisher_html_pipeline() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "enable_protection": true, + "protection_excluded_ip_cidrs": ["192.0.2.0/24"], + "client_side_key": "test-client-key", + }), + ) + .expect("should configure DataDome integration"); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with DataDome"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"content".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_secret_http_client_and_client_ip( + NoopSecretStore, + Arc::clone(&stub) as Arc, + Some("192.0.2.10".parse().expect("should parse client IP")), + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + + let filter_outcome = registry + .filter_request(crate::integrations::RequestFilterRegistryInput { + settings: &settings, + services: &services, + req: &mut req, + geo_info: None, + }) + .await + .expect("should run DataDome filter"); + assert!(matches!( + filter_outcome, + crate::integrations::RequestFilterRegistryOutcome::Continue(_) + )); + let publisher_response = run_publisher_proxy(&settings, &services, req).await; + let response = buffer_publisher_response_async( + publisher_response, + &Method::GET, + &settings, + ®istry, + &AuctionOrchestrator::new(settings.auction.clone()), + &services, + ) + .await + .expect("should buffer publisher response"); + let html = response_body_string(response); + + assert!(!html.contains("window.ddjskey")); + assert!(!html.contains("/integrations/datadome/tags.js")); + assert_eq!( + stub.recorded_backend_names().len(), + 1, + "only the publisher origin should be called" + ); + } + + #[test] + fn suppressed_datadome_tag_reaches_publisher_html_pipeline() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "client_side_key": "test-client-key", + }), + ) + .expect("should configure DataDome integration"); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with DataDome"); + let mut params = make_stream_params(&settings, "identity"); + params.content_type = "text/html; charset=utf-8".to_string(); + params.suppress_datadome_client_side_tag = true; + let mut output = Vec::new(); + + stream_publisher_body( + EdgeBody::from(b"content".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, ) - .await - .expect("should proxy publisher request"); + .expect("should process suppressed HTML"); - assert_eq!( - ec_context.ec_value(), - None, - "handler must not self-generate an EC ID; generation is the adapter's real-browser-gated responsibility", + let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); + assert!( + !html.contains("window.ddjskey"), + "publisher processing should omit the DataDome client configuration" + ); + assert!( + !html.contains("/integrations/datadome/tags.js"), + "publisher processing should omit the DataDome client tag URL" ); } @@ -4852,7 +6014,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build cacheable HTML response"); - apply_datadome_client_tag_cache_privacy( + super::apply_datadome_client_tag_cache_privacy( &mut response, &Method::GET, true, @@ -4867,19 +6029,91 @@ mod tests { Some("no-store, private"), "suppressed HTML should be private and non-storable" ); - for header_name in [ - "surrogate-control", - "fastly-surrogate-control", - "cloudflare-cdn-cache-control", - "cdn-cache-control", - header::ETAG.as_str(), - header::LAST_MODIFIED.as_str(), - ] { + assert!( + response.headers().get("surrogate-control").is_none(), + "suppressed HTML should not retain Surrogate-Control" + ); + assert!( + response.headers().get("fastly-surrogate-control").is_none(), + "suppressed HTML should not retain Fastly-Surrogate-Control" + ); + assert!( + response + .headers() + .get("cloudflare-cdn-cache-control") + .is_none(), + "suppressed HTML should not retain Cloudflare-CDN-Cache-Control" + ); + assert!( + response.headers().get("cdn-cache-control").is_none(), + "suppressed HTML should not retain CDN-Cache-Control" + ); + for header_name in [header::ETAG, header::LAST_MODIFIED] { assert!( - !response.headers().contains_key(header_name), + !response.headers().contains_key(&header_name), "suppressed HTML should not retain {header_name}" ); } + + let mut no_store_response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .body(EdgeBody::empty()) + .expect("should build no-store HTML response"); + super::apply_datadome_client_tag_cache_privacy( + &mut no_store_response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + assert_eq!( + no_store_response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "suppressed HTML should use the exact synthesized-HTML policy" + ); + } + + #[test] + fn datadome_cache_privacy_does_not_change_non_html_or_unsuppressed_responses() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .body(EdgeBody::empty()) + .expect("should build cacheable response"); + + super::apply_datadome_client_tag_cache_privacy( + &mut response, + &Method::GET, + false, + "text/html; charset=utf-8", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "unsuppressed HTML should retain its existing cache policy" + ); + + super::apply_datadome_client_tag_cache_privacy( + &mut response, + &Method::GET, + true, + "text/css", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "non-HTML should retain its existing cache policy" + ); } #[test] @@ -5616,7 +6850,7 @@ mod tests { "https://publisher.example/static/tsjs=unknown.js", ); - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SurrogateControl) + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) .expect("should handle tsjs request"); assert_eq!(response.status(), StatusCode::NOT_FOUND); } @@ -5631,7 +6865,7 @@ mod tests { "https://publisher.example/static/tsjs=tsjs-unified.min.js", ); - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SurrogateControl) + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) .expect("should handle tsjs request"); assert_eq!(response.status(), StatusCode::OK); } @@ -5654,7 +6888,7 @@ mod tests { HeaderValue::from_static("__Host-ts-console=1"), ); - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SurrogateControl) + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) .expect("should handle tsjs request"); assert_eq!(response.status(), StatusCode::OK); @@ -5670,155 +6904,38 @@ mod tests { } #[test] - fn tsjs_dynamic_uses_immutable_cache_for_matching_hash() { - let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let module_ids = registry.js_module_ids_immediate(); - let hash = trusted_server_js::concatenated_hash(&module_ids); - let req = build_request( - Method::GET, - &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), - ); - - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SurrogateControl) - .expect("should handle tsjs request"); - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=31536000, immutable"), - "should make matching content-versioned bundle immutable" - ); - assert_eq!( - response - .headers() - .get("surrogate-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=31536000"), - "should give Fastly edge cache the same immutable TTL" - ); - assert_eq!( - response - .headers() - .get(header::VARY) - .and_then(|value| value.to_str().ok()), - Some("Accept-Encoding"), - "should keep encoding in the cache key" - ); - assert_eq!( - response - .headers() - .get(HEADER_X_COMPRESS_HINT) - .and_then(|value| value.to_str().ok()), - Some("on"), - "should keep Fastly delivery compression hint" - ); - } - - #[test] - fn tsjs_dynamic_uses_cloudflare_edge_header_when_selected() { - let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let module_ids = registry.js_module_ids_immediate(); - let hash = trusted_server_js::concatenated_hash(&module_ids); - let req = build_request( - Method::GET, - &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), - ); - - let response = - handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::CloudflareCdnCacheControl) - .expect("should handle tsjs request"); - - assert_eq!( - response - .headers() - .get("cloudflare-cdn-cache-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=31536000"), - "should render Cloudflare-specific edge cache header" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "Cloudflare responses should not emit Fastly Surrogate-Control" - ); - } - - #[test] - fn tsjs_dynamic_keeps_short_cache_for_mismatched_hash() { - let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let req = build_request( - Method::GET, - "https://publisher.example/static/tsjs=tsjs-unified.min.js?v=not-the-hash", - ); - - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SurrogateControl) - .expect("should handle tsjs request"); - let cache_control = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()) - .expect("should set cache-control"); - - assert_eq!(response.status(), StatusCode::OK); - assert!( - cache_control.contains("max-age=300"), - "should keep short browser TTL for mismatched hash" - ); - assert!( - !cache_control.contains("immutable"), - "should not make mismatched hash requests immutable" - ); - assert_eq!( - response - .headers() - .get("surrogate-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=300, stale-while-revalidate=60, stale-if-error=86400"), - "should keep short edge TTL for mismatched hash" - ); - } - - #[test] - fn parse_deferred_module_filename_extracts_known_id() { + fn parse_single_module_filename_extracts_known_id() { assert_eq!( - parse_deferred_module_filename("tsjs-sourcepoint.min.js"), + parse_single_module_filename("tsjs-sourcepoint.min.js"), Some("sourcepoint"), "should extract sourcepoint from minified filename" ); assert_eq!( - parse_deferred_module_filename("tsjs-sourcepoint.js"), + parse_single_module_filename("tsjs-sourcepoint.js"), Some("sourcepoint"), "should extract sourcepoint from unminified filename" ); } #[test] - fn parse_deferred_module_filename_rejects_unknown_ids() { + fn parse_single_module_filename_rejects_unknown_ids() { assert_eq!( - parse_deferred_module_filename("tsjs-evil.min.js"), + parse_single_module_filename("tsjs-evil.min.js"), None, "should reject unknown module names" ); assert_eq!( - parse_deferred_module_filename("tsjs-core.min.js"), + parse_single_module_filename("tsjs-core.min.js"), Some("core"), "should accept any known module ID (deferred check happens in caller)" ); assert_eq!( - parse_deferred_module_filename("prebid.min.js"), + parse_single_module_filename("prebid.min.js"), None, "should reject without tsjs- prefix" ); assert_eq!( - parse_deferred_module_filename("tsjs-sourcepoint.txt"), + parse_single_module_filename("tsjs-sourcepoint.txt"), None, "should reject non-js extension" ); @@ -5834,12 +6951,12 @@ mod tests { "https://publisher.example/static/tsjs=tsjs-prebid.min.js", ); - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SurrogateControl) + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) .expect("should handle tsjs request"); assert_eq!( response.status(), StatusCode::OK, - "should serve the deferred Prebid shim when Prebid is enabled" + "should serve the deferred prebid shim module when prebid is enabled" ); } @@ -5864,7 +6981,7 @@ mod tests { "https://publisher.example/static/tsjs=tsjs-prebid.min.js", ); - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SurrogateControl) + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) .expect("should handle tsjs request"); assert_eq!( response.status(), @@ -5882,13 +6999,113 @@ mod tests { Method::GET, "https://publisher.example/static/tsjs=tsjs-evil.min.js", ); - - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SurrogateControl) - .expect("should handle tsjs request"); + + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should reject unknown module names" + ); + } + + #[test] + fn tsjs_dynamic_uses_immutable_cache_for_matching_hash() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let module_ids = registry.js_module_ids_immediate(); + let hash = trusted_server_js::concatenated_hash(&module_ids); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), + ); + + let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SurrogateControl) + .expect("should handle tsjs request"); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=31536000, immutable"), + "matching content-versioned bundle should be immutable" + ); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=31536000"), + "matching content-versioned bundle should set the Fastly edge TTL" + ); + } + + #[test] + fn tsjs_dynamic_uses_cloudflare_edge_header_when_selected() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let module_ids = registry.js_module_ids_immediate(); + let hash = trusted_server_js::concatenated_hash(&module_ids); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), + ); + + let response = handle_tsjs_dynamic( + &request, + ®istry, + EdgeCacheHeader::CloudflareCdnCacheControl, + ) + .expect("should handle tsjs request"); + + assert_eq!( + response + .headers() + .get("cloudflare-cdn-cache-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=31536000"), + "Cloudflare requests should use its edge cache header" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "Cloudflare requests should not emit Fastly's edge cache header" + ); + } + + #[test] + fn tsjs_dynamic_keeps_short_cache_for_mismatched_hash() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let request = build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-unified.min.js?v=not-the-hash", + ); + + let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SurrogateControl) + .expect("should handle tsjs request"); + let cache_control = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .expect("should set cache-control"); + + assert_eq!(response.status(), StatusCode::OK); + assert!( + cache_control.contains("max-age=300") && !cache_control.contains("immutable"), + "mismatched hash should retain the short, mutable cache policy" + ); assert_eq!( - response.status(), - StatusCode::NOT_FOUND, - "should reject unknown module names" + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=300, stale-while-revalidate=60, stale-if-error=86400"), + "mismatched hash should retain the short Fastly edge TTL" ); } @@ -5959,8 +7176,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -6008,8 +7225,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -6046,8 +7263,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::from_stream(futures::stream::iter(vec![Ok::<_, io::Error>( bytes::Bytes::from_static(b"live"), @@ -6162,8 +7379,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"body{background:url('https://origin.example.com/"), @@ -6216,8 +7433,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = gzip_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6273,8 +7490,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = deflate_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6330,8 +7547,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6387,8 +7604,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6432,8 +7649,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -6627,8 +7844,8 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"hello"), @@ -6658,7 +7875,7 @@ mod tests { "should still inject ad slots. Got: {html}" ); assert!( - html.contains(".bids=JSON.parse"), + html.contains("var b=JSON.parse("), "should collect auction and inject bids before body close. Got: {html}" ); }); @@ -6692,8 +7909,8 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; // The `` that triggers bid injection lives in the SECOND gzip // member. `flate2::read::GzDecoder` decodes only the first member, so @@ -6726,7 +7943,7 @@ mod tests { "should decode the second gzip member that a single-member decoder drops. Got: {html}" ); assert!( - html.contains(".bids=JSON.parse"), + html.contains("var b=JSON.parse("), "should inject bids before the carried in the second member. Got: {html}" ); }); @@ -6756,8 +7973,8 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( b"body{background:url('https://origin.example.com/asset.png')}", @@ -6813,8 +8030,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let publisher_response = PublisherResponse::Stream { response, @@ -6950,8 +8167,8 @@ mod tests { auction_request: dispatched_auction.as_ref().map(|_| test_auction_request()), dispatched_auction, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -7025,7 +8242,7 @@ mod tests { "prefix must carry the injected (rewritten) head before EOF. Got: {html}" ); assert!( - !html.contains(".bids=JSON.parse"), + !html.contains("var b=JSON.parse("), "bids inject only at after collection, which the first poll must not wait for. Got: {html}" ); } @@ -7067,7 +8284,7 @@ mod tests { "first poll must emit the decoded document prefix of a small gzip page. Got: {decoded}" ); assert!( - !decoded.contains(".bids=JSON.parse"), + !decoded.contains("var b=JSON.parse("), "bids inject only at after collection, which the first poll must not wait for. Got: {decoded}" ); } @@ -7302,8 +8519,8 @@ mod tests { 10, )), price_granularity: PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } }; let make_stream_response = || PublisherResponse::Stream { @@ -7482,8 +8699,8 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let publisher_response = PublisherResponse::Stream { response, @@ -7511,7 +8728,7 @@ mod tests { let html = String::from_utf8(gzip_decode(&output)).expect("should be valid UTF-8"); assert!( - html.contains(".bids=JSON.parse"), + html.contains("var b=JSON.parse("), "should collect the held auction and inject bids. Got tail: {}", &html[html.len().saturating_sub(200)..] ); @@ -7550,8 +8767,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -7601,8 +8818,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let bogus_body = EdgeBody::from(b"not gzip".to_vec()); @@ -7710,8 +8927,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -7768,8 +8985,8 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - suppress_datadome_client_side_tag: false, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -7801,9 +9018,9 @@ mod tests { mod creative_opportunities_tests { use super::super::{ MatchedSlotsContext, build_ad_slots_script, build_auction_request, build_bid_map, - build_bids_script, html_escape_for_script, + build_bids_script, diagnostics_auction_id, html_escape_for_script, write_bids_to_state, }; - use crate::auction::types::{Bid, MediaType}; + use crate::auction::types::{ApsRendererV1, ApsTagType, Bid, BidRenderer, MediaType}; use crate::consent::ConsentContext; use crate::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunityFormat, CreativeOpportunitySlot, @@ -7872,9 +9089,9 @@ mod tests { nurl: Some(nurl.to_string()), burl: Some(burl.to_string()), bid_id: None, - ad_id: Some(ad_id.to_string()), creative_id: None, renderer: None, + ad_id: Some(ad_id.to_string()), cache_id: None, cache_host: None, cache_path: None, @@ -7915,6 +9132,91 @@ mod tests { assert!(!inner.contains('>'), "no unescaped > in script content"); } + #[test] + fn ad_slots_script_omits_only_over_limit_dynamic_slot() { + let mut over_limit = make_slot(); + over_limit.id = "over_limit_dynamic".to_string(); + over_limit.gam_unit_path = Some("/{section}/{section}".to_string()); + over_limit + .compile_unit_template() + .expect("template should compile"); + let mut valid_static = make_slot(); + valid_static.id = "valid_static_sibling".to_string(); + valid_static.gam_unit_path = Some("/12345/example/static".to_string()); + let slots = vec![over_limit, valid_static]; + let config = make_config(); + let request_path = format!("/{}", "a".repeat(60)); + + let script = build_ad_slots_script(&slots, &config, &request_path); + + assert!( + !script.contains("over_limit_dynamic"), + "should omit the over-limit dynamic slot" + ); + assert!( + script.contains("valid_static_sibling"), + "should retain the valid static sibling" + ); + } + + #[test] + fn build_slot_json_renders_section_from_request_path() { + let mut config = make_config(); + config.gam_network_id = "99999".to_string(); + config.section_root = Some("homepage".to_string()); + let mut slot = make_slot(); + slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); + slot.compile_unit_template() + .expect("template should compile"); + + let news_section = config.section_for_path("/news/article-123"); + let news = crate::publisher::build_slot_json(&slot, &config, &news_section) + .expect("should render slot"); + assert_eq!( + news["gam_unit_path"], "/99999/example/news", + "section should derive from the first path segment" + ); + + let home_section = config.section_for_path("/"); + let home = crate::publisher::build_slot_json(&slot, &config, &home_section) + .expect("should render slot"); + assert_eq!( + home["gam_unit_path"], "/99999/example/homepage", + "root path should use section_root" + ); + } + + #[test] + fn build_slot_json_honours_configured_section_segment() { + // Locale-prefixed publisher: `/en/news/article` must resolve to the + // `news` unit, not `en`. + let mut config = make_config(); + config.gam_network_id = "99999".to_string(); + config.section_root = Some("homepage".to_string()); + config.section_segment = Some(1); + let mut slot = make_slot(); + slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); + slot.compile_unit_template() + .expect("template should compile"); + + let news_section = config.section_for_path("/en/news/article-123"); + let news = crate::publisher::build_slot_json(&slot, &config, &news_section) + .expect("should render slot"); + assert_eq!( + news["gam_unit_path"], "/99999/example/news", + "section should derive from the configured segment index" + ); + + let locale_root_section = config.section_for_path("/en"); + let locale_root = + crate::publisher::build_slot_json(&slot, &config, &locale_root_section) + .expect("should render slot"); + assert_eq!( + locale_root["gam_unit_path"], "/99999/example/homepage", + "a path with no segment at the configured index should use section_root" + ); + } + #[test] fn bid_map_includes_nurl_and_burl() { let mut winning_bids = HashMap::new(); @@ -7965,6 +9267,139 @@ mod tests { ); } + /// Guards the browser-visible token every auction path shares: it must + /// be fresh per auction and absent unless diagnostics can consume it. + #[test] + fn diagnostics_auction_id_is_fresh_and_gated() { + let mut settings = test_settings(); + assert_eq!( + diagnostics_auction_id(&settings), + None, + "no token should be minted without the diagnostics integration" + ); + + settings + .integrations + .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) + .expect("should enable diagnostics"); + let first = + diagnostics_auction_id(&settings).expect("enabled diagnostics should mint a token"); + let second = + diagnostics_auction_id(&settings).expect("enabled diagnostics should mint a token"); + + assert!( + first.starts_with("ts-auc-"), + "token should use the diagnostics prefix, got `{first}`" + ); + assert_ne!(first, second, "each auction should mint its own token"); + } + + #[test] + fn initial_document_bids_script_includes_auction_id_only_for_winning_bids() { + let slot = make_slot(); + let slots = [slot]; + let slots_ctx = MatchedSlotsContext { + matched_slots: &slots, + request_path: "/2024/01/my-article/", + }; + let request_info = RequestInfo { + host: "publisher.example.com".to_string(), + scheme: "https".to_string(), + }; + let mut auction_request = build_auction_request( + &slots_ctx, + None, + &ConsentContext::default(), + &request_info, + "publisher.example.com", + Some("Mozilla/5.0"), + ); + auction_request.id = "initial-auction-example-123".to_string(); + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + make_bid( + "atf_sidebar_ad", + 1.50, + "example_bidder", + "abc123", + "https://example.com/win", + "https://example.com/bill", + ), + ); + + let state = std::sync::Arc::new(std::sync::Mutex::new(None)); + write_bids_to_state( + &winning_bids, + PriceGranularity::Dense, + &state, + &test_settings(), + "", + false, + Some(&auction_request.id), + ); + let script = state + .lock() + .expect("should lock initial bid state") + .clone() + .expect("should generate initial-document bids script"); + let bid_json = script + .strip_prefix( + "", + ) + }) + .expect("should emit the initial-document tsjs.bids script shape"); + let bid_json: String = serde_json::from_str(&format!("\"{bid_json}\"")) + .expect("should decode initial-document JSON.parse input"); + let bids: serde_json::Value = serde_json::from_str(&bid_json) + .expect("should serialize initial-document bids as JSON"); + + assert_eq!( + bids["atf_sidebar_ad"]["hb_auction_id"], auction_request.id, + "initial-document bids should expose the current request ID only on the winner" + ); + + write_bids_to_state( + &HashMap::new(), + PriceGranularity::Dense, + &state, + &test_settings(), + "", + false, + Some(&auction_request.id), + ); + let empty_script = state + .lock() + .expect("should lock empty initial bid state") + .clone() + .expect("should generate empty initial-document bids script"); + let empty_bid_json = empty_script + .strip_prefix( + "", + ) + }) + .expect("should emit the empty initial-document tsjs.bids script shape"); + let empty_bid_json: String = serde_json::from_str(&format!("\"{empty_bid_json}\"")) + .expect("should decode empty initial-document JSON.parse input"); + let empty_bids: serde_json::Value = serde_json::from_str(&empty_bid_json) + .expect("should serialize empty initial-document bids as JSON"); + assert!( + empty_bids + .as_object() + .expect("initial-document bids should be an object") + .is_empty(), + "initial-document bids should not fabricate metadata without a winner" + ); + } + #[test] fn bid_map_omits_zero_creative_dimensions() { // Missing OpenRTB w/h parse to 0. Emitting w:0/h:0 would make the @@ -8082,10 +9517,11 @@ mod tests { #[test] fn build_bid_map_sanitizes_hostile_adm() { - // The inline-adm path must run the same creative-processing boundary - // as the `/auction` path (sanitize → rewrite) before the creative - // reaches window.tsjs.bids, so hostile executable markup never lands - // in the client-facing `adm` for the Prebid Universal Creative to run. + // The inline-adm path must run the same opt-in creative-processing + // boundary as the `/auction` path (sanitize → rewrite) before the + // creative reaches window.tsjs.bids, so with sanitization enabled + // hostile executable markup never lands in the client-facing `adm` + // for the Prebid Universal Creative to run. let mut settings = test_settings(); settings.auction.sanitize_creatives = true; let mut winning_bids = HashMap::new(); @@ -8131,10 +9567,10 @@ mod tests { } #[test] - fn build_bid_map_can_skip_rewriting_but_not_sanitization() { + fn build_bid_map_can_skip_rewriting_while_sanitizing() { let mut settings = test_settings(); - settings.auction.rewrite_creatives = false; settings.auction.sanitize_creatives = true; + settings.auction.rewrite_creatives = false; let mut winning_bids = HashMap::new(); let mut bid = make_bid( "atf_sidebar_ad", @@ -8190,10 +9626,11 @@ mod tests { #[test] fn build_bid_map_omits_oversized_adm() { - // Creatives larger than the sanitize pass's 1 MiB cap are rejected - // (empty result), so the inline `adm` is omitted and the pbRender - // bridge falls back to the PBS Cache coordinates instead of shipping - // an unbounded creative to the client. + // Creatives larger than the 1 MiB cap are rejected (empty result) + // in every processing mode, so the bid is omitted rather than + // recording a blank winner or shipping an unbounded creative to the + // client. Runs with default settings to cover the shipped + // configuration. let mut winning_bids = HashMap::new(); let mut bid = make_bid( "atf_sidebar_ad", @@ -8206,6 +9643,110 @@ mod tests { bid.creative = Some(format!("
{}
", "a".repeat(1024 * 1024 + 1))); winning_bids.insert("atf_sidebar_ad".to_string(), bid); + let map = build_bid_map( + &winning_bids, + PriceGranularity::Dense, + &test_settings(), + "", + false, + ); + assert!( + !map.contains_key("atf_sidebar_ad"), + "should omit the bid when the creative exceeds the 1 MiB cap" + ); + } + + // A supplied creative that processing rejects must not fall back to the + // PBS Cache coordinates: the GPT bridge fetches the cached bid's ORIGINAL + // adm, which would undo sanitization and the size cap entirely. + fn cached_bid_with_creative(creative: &str) -> Bid { + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: Some(creative.to_string()), + adomain: None, + bidder: "prebid".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("bid-impression-id".to_string()), + bid_id: Some("openrtb-bid-id".to_string()), + creative_id: None, + // No typed renderer: these cases assert what happens when the + // supplied markup is the bid's only render source. + renderer: None, + cache_id: Some("cache-uuid".to_string()), + cache_host: Some("prebid-cache.example.com".to_string()), + cache_path: Some("/cache".to_string()), + metadata: Default::default(), + } + } + + // These fixtures carry cache coordinates but no typed renderer, so a + // rejected creative leaves the bid with no render source at all and it + // is dropped outright — which subsumes the property under test: the + // cache coordinates never reach the client, so the cached (unprocessed) + // copy of the markup cannot be fetched in place of what was refused. + fn assert_no_render_source(settings: &Settings, creative: String, case: &str) { + let mut winning_bids = HashMap::new(); + let mut bid = cached_bid_with_creative(""); + bid.creative = Some(creative); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, settings, "", false); + + match map.get("atf_sidebar_ad").and_then(|v| v.as_object()) { + None => {} + Some(obj) => { + assert!( + obj.get("adm").is_none(), + "{case}: rejected creative should not emit adm" + ); + assert!( + obj.get("hb_cache_host").is_none(), + "{case}: rejected creative should suppress hb_cache_host" + ); + assert!( + obj.get("hb_cache_path").is_none(), + "{case}: rejected creative should suppress hb_cache_path" + ); + } + } + } + + #[test] + fn build_bid_map_suppresses_cache_fallback_for_rejected_creatives() { + let mut sanitizing = test_settings(); + sanitizing.auction.sanitize_creatives = true; + + // Script-only creative: sanitization strips everything. + assert_no_render_source( + &sanitizing, + "".to_string(), + "script-only", + ); + // Oversized creative: rejected by the cap in every mode. + assert_no_render_source( + &test_settings(), + format!("
{}
", "a".repeat(1024 * 1024 + 1)), + "oversized", + ); + // An explicit empty `adm` is a supplied creative, not an absent one: + // classifying it as absent would re-enable the raw cache fallback. + assert_no_render_source(&test_settings(), String::new(), "explicit-empty"); + } + + #[test] + fn build_bid_map_keeps_cache_fallback_for_absent_creatives() { + // A bid with no supplied creative is the legitimate PBS Cache case: + // the coordinates are the only render source. + let mut winning_bids = HashMap::new(); + let mut bid = cached_bid_with_creative(""); + bid.creative = None; + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + let map = build_bid_map( &winning_bids, PriceGranularity::Dense, @@ -8217,9 +9758,16 @@ mod tests { .get("atf_sidebar_ad") .and_then(|v| v.as_object()) .expect("should have a bid entry"); - assert!( - obj.get("adm").is_none(), - "should omit the inline adm when the creative exceeds the 1 MiB cap" + + assert_eq!( + obj.get("hb_cache_host").and_then(|v| v.as_str()), + Some("prebid-cache.example.com"), + "absent creative should keep hb_cache_host" + ); + assert_eq!( + obj.get("hb_cache_path").and_then(|v| v.as_str()), + Some("/cache"), + "absent creative should keep hb_cache_path" ); } @@ -8231,6 +9779,7 @@ mod tests { // root-relative `/first-party/proxy` would resolve against GAM and 404. // The tsjs bundle must NOT be injected into that foreign-origin iframe. let mut settings = test_settings(); + settings.auction.rewrite_creatives = true; settings.publisher.domain = "example.com".to_string(); let mut winning_bids = HashMap::new(); @@ -8280,6 +9829,7 @@ mod tests { // origin the visitor is on (here an HTTP dev host with a port), not the // configured publisher domain. let mut settings = test_settings(); + settings.auction.rewrite_creatives = true; settings.publisher.domain = "example.com".to_string(); let mut winning_bids = HashMap::new(); @@ -8525,9 +10075,9 @@ mod tests { nurl: None, burl: None, bid_id: None, - ad_id: Some("bid-impression-id".to_string()), creative_id: None, renderer: None, + ad_id: Some("bid-impression-id".to_string()), cache_id: Some("f47447a0-b759-4f2f-9887-af458b79b570".to_string()), cache_host: Some("openads.adsrvr.org".to_string()), cache_path: Some("/cache".to_string()), @@ -8616,6 +10166,65 @@ mod tests { ); } + #[test] + fn bid_map_exposes_aps_renderer_and_selected_bid_id() { + // Sanitization is opt-in, so enable it: the script-only creative + // below is what drives this bid onto the renderer path. Left at the + // default it would survive processing as an ordinary creative. + let mut settings = test_settings(); + settings.auction.sanitize_creatives = true; + let mut bid = make_bid("atf_sidebar_ad", 1.50, "aps", "fallback-ad", "", ""); + bid.bid_id = Some("selected-bid".to_string()); + bid.creative = Some("".to_string()); + bid.nurl = None; + bid.burl = None; + bid.renderer = Some(BidRenderer::Aps(ApsRendererV1 { + version: 1, + account_id: "example-account".to_string(), + bid_id: "selected-bid".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "fictional-base64".to_string(), + width: 300, + height: 250, + })); + let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); + let obj = map["atf_sidebar_ad"] + .as_object() + .expect("should include APS bid"); + + assert_eq!(obj["hb_bidder"], "aps"); + assert_eq!(obj["hb_adid"], "selected-bid"); + assert_eq!(obj["renderer"]["type"], "aps"); + assert_eq!(obj["renderer"]["bidId"], "selected-bid"); + assert!(obj.get("adm").is_none()); + + let script = build_bids_script(&map); + assert!(!script.contains("")); + assert!(script.contains("\\u003C/script\\u003E")); + } + + #[test] + fn bid_map_omits_creative_rejected_by_processing_without_renderer() { + // Sanitization is opt-in, so enable it: script-only markup is what + // makes processing reject this bid's only render source. + let mut settings = test_settings(); + settings.auction.sanitize_creatives = true; + let mut bid = make_bid("atf_sidebar_ad", 1.50, "kargo", "fallback-ad", "", ""); + bid.creative = Some("".to_string()); + let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); + + assert!( + !map.contains_key("atf_sidebar_ad"), + "should omit a bid whose only creative was rejected" + ); + } + #[test] fn bid_map_omits_hb_adid_when_both_cache_id_and_ad_id_absent() { let mut winning_bids = HashMap::new(); @@ -8633,9 +10242,9 @@ mod tests { nurl: None, burl: None, bid_id: None, - ad_id: None, creative_id: None, renderer: None, + ad_id: None, cache_id: None, cache_host: None, cache_path: None, @@ -8677,9 +10286,9 @@ mod tests { nurl: None, burl: None, bid_id: None, - ad_id: None, creative_id: None, renderer: None, + ad_id: None, cache_id: None, cache_host: None, cache_path: None, @@ -8712,24 +10321,72 @@ mod tests { } #[test] - fn bids_script_calls_ad_init_without_retry_timer() { + fn bids_script_schedules_ad_init_without_retry_timer() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + + let script = build_bids_script(&map); + + assert!( + script.contains("t.scheduleInitialAdInit"), + "should hand off bids to the deferred adInit scheduler" + ); + assert!( + !script.contains("setTimeout"), + "should not retry adInit on a timer" + ); + assert!( + !script.contains("prevGptSlots"), + "should not use TS-owned slots as adInit success signal" + ); + } + + #[test] + fn bids_script_defers_ad_init_until_after_hydration() { let mut map = serde_json::Map::new(); map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); let script = build_bids_script(&map); + // adInit() mutates ad-slot subtrees (GPT defineSlot on the + // `-container` wrapper). Running it synchronously at body-parse time + // lands those mutations inside React's hydration window and trips a + // #418 hydration mismatch. The deferral lifecycle (window `load`, + // double `requestAnimationFrame`, generation-0 pinning via + // `tsjs.navGeneration`) lives in the GPT bundle module (with a + // head-injected fallback in gpt_bootstrap.js) where it is executable + // under Vitest (schedule_initial_ad_init.test.ts); this inline + // script must only delegate to that scheduler. + assert!( + script.contains("var s=t.scheduleInitialAdInit"), + "should delegate deferral to the installed scheduler" + ); + // The bids payload is handed to the scheduler (which applies it only + // while the page is still on navigation generation 0) instead of + // being assigned unconditionally, so a faster SPA navigation's live + // bids cannot be clobbered by the stale SSR payload. + assert!( + script.contains("if(typeof s===\"function\")s(b)"), + "should pass the SSR bids payload to the scheduler" + ); + assert!( + script.contains("else t.bids=b"), + "should fall back to a plain bids assignment without a scheduler" + ); + assert!( + !script.contains(".bids=JSON.parse"), + "should not assign the SSR payload unconditionally" + ); + // The one hydration-unsafe thing this script could do is invoke + // adInit synchronously at body-parse time — it must not. assert!( - script.contains("window.tsjs.adInit"), - "should hand off bids to adInit" + !script.contains("adInit()"), + "should not invoke adInit synchronously at parse time" ); assert!( !script.contains("setTimeout"), "should not retry adInit on a timer" ); - assert!( - !script.contains("prevGptSlots"), - "should not use TS-owned slots as adInit success signal" - ); } #[test] @@ -8799,12 +10456,58 @@ mod tests { ); assert_eq!( request.publisher.page_url.as_deref(), - Some("https://www.example.com/2024/01/my-article/?edition=fictional"), - "page_url host should be the configured publisher domain, not the edge host" + Some("https://www.example.com/2024/01/my-article/"), + "page_url should use configured publisher identity without client query data" + ); + assert_eq!( + site.page, "https://www.example.com/2024/01/my-article/", + "site.page should use configured publisher identity without client query data" + ); + } + + #[test] + fn auction_request_preserves_configured_publisher_domain_with_query() { + // On the SSAT proxy path the browser addresses the trusted-server + // edge host, but the auction must advertise the configured + // publisher domain to SSPs — otherwise injected creatives and the + // brand-safety pixel leak the edge/staging host. + let slot = make_slot(); + let slots = [slot]; + let slots_ctx = MatchedSlotsContext { + matched_slots: &slots, + request_path: "/2024/01/my-article/?edition=fictional", + }; + let request_info = RequestInfo { + host: "ts.example.com".to_string(), + scheme: "https".to_string(), + }; + + let request = build_auction_request( + &slots_ctx, + None, + &ConsentContext::default(), + &request_info, + "www.example.com", + Some("Mozilla/5.0"), + ); + + assert_eq!( + request.publisher.domain, "www.example.com", + "publisher.domain should be the configured publisher domain, not the edge host" + ); + let site = request.site.expect("should populate site metadata"); + assert_eq!( + site.domain, "www.example.com", + "site.domain should be the configured publisher domain, not the edge host" ); assert_eq!( - site.page, "https://www.example.com/2024/01/my-article/?edition=fictional", - "site.page host should be the configured publisher domain, not the edge host" + request.publisher.page_url.as_deref(), + Some("https://www.example.com/2024/01/my-article/"), + "page_url should remove client query data" + ); + assert_eq!( + site.page, "https://www.example.com/2024/01/my-article/", + "site.page should remove client query data" ); } @@ -8888,11 +10591,108 @@ mod tests { mod page_bids_no_match_tests { use super::super::*; + use super::build_services_with_http_client; use crate::auction::AuctionOrchestrator; + use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::types::{AuctionRequest, AuctionResponse, Bid}; use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; - use crate::platform::test_support::noop_services; + use crate::platform::test_support::{StubHttpClient, noop_services}; + use crate::platform::{PlatformHttpRequest, PlatformResponse}; use crate::test_support::tests::crate_test_settings_str; + use error_stack::{Report, ResultExt}; use http::Method; + use std::sync::{Arc, Mutex}; + + const AUCTION_ID_TEST_PROVIDER: &str = "auction_id_test_provider"; + const AUCTION_ID_TEST_BACKEND: &str = "auction-id-test-backend"; + + struct AuctionIdTestProvider { + captured_request: Arc>>, + winning_bid: bool, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for AuctionIdTestProvider { + fn provider_name(&self) -> &'static str { + AUCTION_ID_TEST_PROVIDER + } + + async fn request_bids( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + *self + .captured_request + .lock() + .expect("should lock captured auction request") = Some(request.clone()); + let request = PlatformHttpRequest::new( + Request::builder() + .method(Method::POST) + .uri("https://bidder.example.test/bids") + .body(EdgeBody::empty()) + .expect("should build test bidder request"), + AUCTION_ID_TEST_BACKEND, + ); + context + .services + .http_client() + .send_async(request) + .await + .change_context(TrustedServerError::Auction { + message: "test bidder launch failed".to_string(), + }) + .map(ProviderRequestOutcome::pending) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + let bids = if self.winning_bid { + vec![Bid { + slot_id: "atf".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: AUCTION_ID_TEST_PROVIDER.to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + bid_id: None, + creative_id: None, + renderer: None, + ad_id: Some("winner-123".to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }] + } else { + Vec::new() + }; + Ok(AuctionResponse::success( + AUCTION_ID_TEST_PROVIDER, + bids, + response_time_ms, + )) + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name( + &self, + _services: &RuntimeServices, + _timeout_ms: u32, + ) -> Option { + Some(AUCTION_ID_TEST_BACKEND.to_string()) + } + } fn settings_with_co() -> Settings { let toml = format!( @@ -9037,6 +10837,206 @@ mod tests { .expect("should return ok response") } + fn auction_id_test_orchestrator( + settings: &Settings, + captured_request: Arc>>, + winning_bid: bool, + ) -> AuctionOrchestrator { + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(AuctionIdTestProvider { + captured_request, + winning_bid, + })); + orchestrator + } + + #[tokio::test] + async fn page_bids_response_includes_auction_id_only_for_winning_bids() { + let mut settings = settings_with_co(); + settings.auction.providers = vec![AUCTION_ID_TEST_PROVIDER.to_string()]; + settings + .integrations + .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) + .expect("should enable diagnostics"); + let slots = article_slot(); + let winning_stub = Arc::new(StubHttpClient::new()); + winning_stub.push_response(200, b"winner".to_vec()); + let winning_services = build_services_with_http_client( + Arc::clone(&winning_stub) as Arc + ); + let winning_request = Arc::new(Mutex::new(None)); + let winning_orchestrator = + auction_id_test_orchestrator(&settings, Arc::clone(&winning_request), true); + let ec_context = EcContext::new_for_test( + Some("page-auction-example-123".to_string()), + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }, + ); + + let winning_response = handle_page_bids( + &settings, + &winning_services, + None, + AuctionDispatch { + orchestrator: &winning_orchestrator, + slots: &slots, + registry: None, + }, + &ec_context, + make_page_bids_request("/2024/01/my-article/"), + ) + .await + .expect("should return winning page-bids response"); + let winning_body: serde_json::Value = serde_json::from_slice( + &winning_response + .into_body() + .into_bytes() + .expect("should read winning page-bids response body"), + ) + .expect("should serialize winning page-bids response as JSON"); + let auction_request = winning_request + .lock() + .expect("should lock captured winning request") + .clone() + .expect("should dispatch a winning auction request"); + + assert_eq!( + auction_request.id, "ts-page-auction-example-123", + "test EC ID should produce a deterministic auction request ID" + ); + let winning_auction_id = winning_body["bids"]["atf"]["hb_auction_id"] + .as_str() + .expect("page-bids should expose an auction ID on the winner") + .to_string(); + assert!( + winning_auction_id.starts_with("ts-auc-"), + "page-bids should expose a freshly minted diagnostics token, got `{winning_auction_id}`" + ); + assert_ne!( + winning_auction_id, auction_request.id, + "browser-visible auction ID must not be the EC-derived request ID" + ); + assert!( + !winning_auction_id.contains("page-auction-example-123"), + "browser-visible auction ID must not embed the EC ID" + ); + + let no_winner_stub = Arc::new(StubHttpClient::new()); + no_winner_stub.push_response(200, b"no-bid".to_vec()); + let no_winner_services = build_services_with_http_client( + Arc::clone(&no_winner_stub) as Arc + ); + let no_winner_orchestrator = + auction_id_test_orchestrator(&settings, Arc::new(Mutex::new(None)), false); + let no_winner_response = handle_page_bids( + &settings, + &no_winner_services, + None, + AuctionDispatch { + orchestrator: &no_winner_orchestrator, + slots: &slots, + registry: None, + }, + &ec_context, + make_page_bids_request("/2024/01/my-article/"), + ) + .await + .expect("should return no-winner page-bids response"); + let no_winner_body: serde_json::Value = serde_json::from_slice( + &no_winner_response + .into_body() + .into_bytes() + .expect("should read no-winner page-bids response body"), + ) + .expect("should serialize no-winner page-bids response as JSON"); + + assert!( + no_winner_body["bids"] + .as_object() + .expect("page-bids should return a bids object") + .is_empty(), + "page-bids should not fabricate auction metadata without a winner" + ); + } + + /// The browser-visible auction ID is minted per auction and only for + /// deployments that run the diagnostics integration, so it can neither + /// carry EC identity across auctions nor reach pages that ignore it. + #[tokio::test] + async fn page_bids_auction_id_is_per_auction_and_gated_on_diagnostics() { + async fn winning_auction_id(settings: &Settings) -> Option { + let slots = article_slot(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"winner".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let orchestrator = + auction_id_test_orchestrator(settings, Arc::new(Mutex::new(None)), true); + let ec_context = EcContext::new_for_test( + Some("page-auction-example-123".to_string()), + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }, + ); + let response = handle_page_bids( + settings, + &services, + None, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &slots, + registry: None, + }, + &ec_context, + make_page_bids_request("/2024/01/my-article/"), + ) + .await + .expect("should return page-bids response"); + let body: serde_json::Value = serde_json::from_slice( + &response + .into_body() + .into_bytes() + .expect("should read page-bids response body"), + ) + .expect("should serialize page-bids response as JSON"); + body["bids"]["atf"]["hb_auction_id"] + .as_str() + .map(str::to_string) + } + + let mut settings = settings_with_co(); + settings.auction.providers = vec![AUCTION_ID_TEST_PROVIDER.to_string()]; + settings + .integrations + .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) + .expect("should enable diagnostics"); + + let first = winning_auction_id(&settings) + .await + .expect("first auction should expose a diagnostics token"); + let second = winning_auction_id(&settings) + .await + .expect("second auction should expose a diagnostics token"); + assert_ne!( + first, second, + "each auction for the same visitor should mint its own token" + ); + + settings + .integrations + .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": false })) + .expect("should disable diagnostics"); + assert_eq!( + winning_auction_id(&settings).await, + None, + "no auction metadata should reach the page without the diagnostics integration" + ); + } + /// The deprecated `/__ts/page-bids` alias must be handled identically to /// the canonical path — same status, same JSON body. /// @@ -9337,6 +11337,46 @@ mod tests { ); } + #[tokio::test] + async fn page_bids_omits_only_over_limit_dynamic_slot() { + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut over_limit = article_slot() + .into_iter() + .next() + .expect("should build over-limit slot"); + over_limit.id = "over_limit_dynamic".to_string(); + over_limit.page_patterns = vec!["/*".to_string()]; + over_limit.gam_unit_path = Some("/{section}/{section}".to_string()); + over_limit + .compile_unit_template() + .expect("template should compile"); + let mut valid_static = article_slot() + .into_iter() + .next() + .expect("should build valid static slot"); + valid_static.id = "valid_static_sibling".to_string(); + valid_static.page_patterns = vec!["/*".to_string()]; + valid_static.gam_unit_path = Some("/12345/example/static".to_string()); + let slots = vec![over_limit, valid_static]; + let request_path = format!("/{}", "a".repeat(60)); + let mut req = make_page_bids_request(&request_path); + set_test_header(&mut req, "sec-purpose", "prefetch"); + + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; + let returned_slots = body["slots"].as_array().expect("slots should be array"); + + assert_eq!( + returned_slots.len(), + 1, + "should omit only the over-limit dynamic slot" + ); + assert_eq!( + returned_slots[0]["id"], "valid_static_sibling", + "should retain the valid static sibling" + ); + } + #[tokio::test] async fn url_not_matching_any_pattern_returns_empty_response() { // Slots exist but request path does not match — no auction, no injection. @@ -9620,6 +11660,38 @@ mod tests { }] } + fn slots_with_over_limit_dynamic_sibling() -> Vec { + let mut over_limit = article_slot() + .into_iter() + .next() + .expect("should build over-limit slot"); + over_limit.id = "over_limit_dynamic".to_string(); + over_limit.page_patterns = vec!["/*".to_string()]; + over_limit.gam_unit_path = Some("/{section}/{section}".to_string()); + + let mut valid_static = article_slot() + .into_iter() + .next() + .expect("should build valid static slot"); + valid_static.id = "valid_static_sibling".to_string(); + valid_static.page_patterns = vec!["/*".to_string()]; + valid_static.gam_unit_path = Some("/12345/example/static".to_string()); + + vec![over_limit, valid_static] + } + + fn assert_only_renderable_slot_was_auctioned( + captured: &Arc>>, + ) { + let request = captured + .lock() + .expect("should lock captured request") + .clone() + .expect("should dispatch an auction request"); + let slot_ids: Vec<_> = request.slots.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(slot_ids, vec!["valid_static_sibling"]); + } + /// [`EcContext`] whose consent context permits the server-side auction. fn consent_allowing_ec_context() -> EcContext { let consent = crate::consent::ConsentContext { @@ -9746,7 +11818,7 @@ mod tests { registry: None, }, req, - EdgeCacheHeader::SurrogateControl, + EdgeCacheHeader::SMaxageFallback, ) .await .expect("should proxy publisher request"); @@ -9795,5 +11867,91 @@ mod tests { assert_configured_domain(&captured, &telemetry_sink); } + + #[tokio::test] + async fn initial_navigation_auctions_only_renderable_slots() { + let settings = settings_with_capturing_provider(); + let captured = Arc::new(Mutex::new(None)); + let orchestrator = orchestrator_capturing_request(&settings, &captured); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = services_with( + Arc::clone(&stub) as Arc, + telemetry_sink, + ); + let mut ec_context = consent_allowing_ec_context(); + let request_path = format!("/{}", "a".repeat(60)); + let req = HttpRequest::builder() + .method(Method::GET) + .uri(format!("https://{EDGE_HOST}{request_path}")) + .header(header::HOST, EDGE_HOST) + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build test request"); + let slots = slots_with_over_limit_dynamic_sibling(); + + let _ = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &slots, + registry: None, + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + + assert_only_renderable_slot_was_auctioned(&captured); + } + + #[tokio::test] + async fn page_bids_auctions_only_renderable_slots() { + let settings = settings_with_capturing_provider(); + let captured = Arc::new(Mutex::new(None)); + let orchestrator = orchestrator_capturing_request(&settings, &captured); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with( + Arc::new(crate::platform::test_support::NoopHttpClient), + telemetry_sink, + ); + let ec_context = consent_allowing_ec_context(); + let request_path = format!("/{}", "a".repeat(60)); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri(format!( + "https://{EDGE_HOST}/_ts/page-bids?path={request_path}" + )) + .header(header::HOST, EDGE_HOST) + .body(EdgeBody::empty()) + .expect("should build test request"); + req.headers_mut().insert( + header::HeaderName::from_static("sec-fetch-site"), + HeaderValue::from_static("same-origin"), + ); + let slots = slots_with_over_limit_dynamic_sibling(); + + let _ = handle_page_bids( + &settings, + &services, + None, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &slots, + registry: None, + }, + &ec_context, + req, + ) + .await + .expect("should return ok response"); + + assert_only_renderable_slot_was_auctioned(&captured); + } } } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index cd5f7f8cd..d4bd21ab1 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2100,13 +2100,26 @@ impl CacheAssetRule { let preset_is_content_addressed = matches!(self.preset, Some(CacheAssetPreset::NextJsStatic)); - if !preset_is_content_addressed && self.fingerprint_style.is_none() { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "cache.asset_rules `{}` sets immutable without fingerprint_style or a content-addressed preset", - self.id - ), - })); + if !preset_is_content_addressed { + match self.fingerprint_style { + None => { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` sets immutable without fingerprint_style or a content-addressed preset", + self.id + ), + })); + } + Some(CacheAssetFingerprintStyle::ViteBase64Url) => { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` cannot set immutable with vite-base64-url; use a content-addressed preset or an unambiguous fingerprint_style", + self.id + ), + })); + } + Some(_) => {} + } } Ok(()) @@ -2222,15 +2235,30 @@ fn compile_cache_asset_glob_patterns( pattern: &str, compiled: &mut Vec, ) -> Result<(), String> { - compiled.push(Pattern::new(pattern).map_err(|err| err.to_string())?); + let mut variants = vec![pattern.to_string()]; + let mut variant_index = 0; + + while variant_index < variants.len() { + let variant = variants[variant_index].clone(); + let optional_segments = variant + .match_indices("**/") + .map(|(index, _)| index) + .collect::>(); + for segment_start in optional_segments { + let without_segment = format!( + "{}{}", + &variant[..segment_start], + &variant[segment_start + "**/".len()..] + ); + if !variants.contains(&without_segment) { + variants.push(without_segment); + } + } + variant_index += 1; + } - if let Some(optional_recursive_start) = pattern.find("**/") { - let without_recursive_segment = format!( - "{}{}", - &pattern[..optional_recursive_start], - &pattern[optional_recursive_start + "**/".len()..] - ); - compile_cache_asset_glob_patterns(&without_recursive_segment, compiled)?; + for variant in variants { + compiled.push(Pattern::new(&variant).map_err(|err| err.to_string())?); } Ok(()) @@ -2279,7 +2307,7 @@ fn path_extension(path: &str) -> Option { (!extension.is_empty()).then(|| extension.to_ascii_lowercase()) } -/// Operator-selected filename fingerprint convention for an immutable custom rule. +/// Operator-selected filename fingerprint convention for a cache rule. #[derive(Debug, Clone, Copy, Deserialize, Serialize, Eq, PartialEq)] #[serde(rename_all = "kebab-case")] pub enum CacheAssetFingerprintStyle { @@ -2287,7 +2315,7 @@ pub enum CacheAssetFingerprintStyle { Hex, /// An eight-character uppercase Base32 suffix, such as `app-VRTVD5R5.js`. EsbuildBase32, - /// An eight-character `Base64URL` suffix, such as `index-BsELY24f.js`. + /// An eight-character `Base64URL` suffix for non-immutable rules, such as `index-BsELY24f.js`. ViteBase64Url, } @@ -3268,11 +3296,6 @@ mod tests { "/assets/app-VRTVD5R5.js", "/assets/index-BsELY24f.js", ), - ( - "vite-base64-url", - "/assets/index-BsELY24f.js", - "/assets/app.0123abcd.js", - ), ] { let toml_str = format!( r#"{} @@ -3309,57 +3332,63 @@ mod tests { } #[test] - fn filename_fingerprint_gate_matches_only_the_selected_style() { - for (style, path, expected) in [ - ( - CacheAssetFingerprintStyle::Hex, - "/assets/app.0123abcd.js", - true, - ), - ( - CacheAssetFingerprintStyle::Hex, - "/assets/hero-Portrait.jpg", - false, - ), - ( - CacheAssetFingerprintStyle::EsbuildBase32, - "/assets/app-VRTVD5R5.js", - true, - ), - ( - CacheAssetFingerprintStyle::EsbuildBase32, - "/assets/hero-Portrait.jpg", - false, - ), - ( - CacheAssetFingerprintStyle::ViteBase64Url, - "/assets/index-BsELY24f.js", - true, - ), - ( - CacheAssetFingerprintStyle::ViteBase64Url, - "/assets/hero-Portrait.jpg", - true, - ), - ( - CacheAssetFingerprintStyle::ViteBase64Url, - "/assets/app.js", - false, - ), - ( - CacheAssetFingerprintStyle::ViteBase64Url, - "/assets/deadbeef.js", - false, - ), + fn immutable_vite_style_cannot_cache_human_named_assets() { + let rule = format!( + r#"{} + + [[cache.asset_rules]] + id = "vite-assets" + enabled = true + path_globs = ["/assets/**/*.js", "/assets/**/*.jpg", "/assets/**/*.png", "/assets/**/*.svg"] + fingerprint_style = "vite-base64-url" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + ); + + for path in [ + "/assets/hero-Portrait.jpg", + "/assets/logo-DarkMode.svg", + "/assets/banner-Summer24.png", ] { - assert_eq!( - filename_contains_fingerprint(path, style), - expected, - "{style:?} fingerprint result should match for {path}" + let error = Settings::from_toml(&rule) + .expect_err("should reject immutable Vite-style cache rule"); + assert!( + format!("{error:?}").contains("cannot set immutable with vite-base64-url"), + "{path} must not receive an immutable policy through a Vite-style rule" ); } } + #[test] + fn non_immutable_vite_style_remains_available_for_cache_matching() { + let toml = format!( + r#"{} + + [[cache.asset_rules]] + id = "vite-assets" + enabled = true + path_glob = "/assets/*.js" + fingerprint_style = "vite-base64-url" + browser_ttl_seconds = 300 + "#, + crate_test_settings_str() + ); + let settings = + Settings::from_toml(&toml).expect("should allow Vite-style matching without immutable"); + + assert!( + settings + .asset_cache_policy_for_path("/assets/index-BsELY24f.js") + .expect("should evaluate Vite-style cache rule") + .is_some(), + "non-immutable Vite-style rule should still match a Vite output filename" + ); + } + #[test] fn cache_asset_rule_globs_respect_path_separators() { let toml_str = format!( @@ -3406,6 +3435,32 @@ mod tests { } } + #[test] + fn cache_asset_rule_globs_expand_each_optional_recursive_segment() { + let toml = format!( + r#"{} + + [[cache.asset_rules]] + id = "nested-assets" + enabled = true + path_glob = "/a/**/b/**/c.js" + browser_ttl_seconds = 300 + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("should parse recursive cache rule"); + + for path in ["/a/x/b/y/c.js", "/a/b/y/c.js", "/a/x/b/c.js", "/a/b/c.js"] { + assert!( + settings + .asset_cache_policy_for_path(path) + .expect("should evaluate recursive cache rule") + .is_some(), + "recursive pattern should match {path}" + ); + } + } + #[test] fn disabled_cache_asset_rules_defer_matcher_and_policy_validation() { let toml_str = format!( diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index d691d8583..860ee0621 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1069,25 +1069,21 @@ at least one of `browser_ttl_seconds` or `edge_ttl_seconds`; private rules must configure `browser_ttl_seconds` and must not configure `edge_ttl_seconds`. `path_glob` and `path_globs` are mutually exclusive. `immutable = true` additionally requires a positive browser TTL and either the content-addressed -`nextjs-static` preset or an explicit `fingerprint_style`. +`nextjs-static` preset, `hex`, or `esbuild-base32`. -The filename fingerprint check is intentionally conservative and style-specific. -It examines the suffix immediately before the final extension and requires a -nonempty filename prefix separated by `.`, `-`, `_`, or `~`. Set exactly the -style emitted by the publisher's bundler: +The filename fingerprint check examines the suffix immediately before the final +extension and requires a nonempty filename prefix separated by `.`, `-`, `_`, +or `~`. The accepted immutable conventions are: - `hex`: hexadecimal suffixes of at least eight characters containing a letter, such as `app.0123abcd.js`; - `esbuild-base32`: eight-character uppercase Base32 suffixes, such as - `app-VRTVD5R5.js`; -- `vite-base64-url`: eight-character Base64URL suffixes with a mixed character - class, such as `index-BsELY24f.js`. + `app-VRTVD5R5.js`. -A style is an explicit operator assertion, not proof of content addressing. -For example, some human-written mixed-case names can resemble a Vite suffix, so -only select `vite-base64-url` after verifying the publisher's build output. A -base rule that matches while its selected fingerprint style fails emits a debug -log with the rule ID and rejected path. +`vite-base64-url` remains available for non-immutable cache rules, but it cannot +prove content addressing. Ordinary names such as `hero-Portrait.jpg` can match +its eight-character Base64URL shape. A matching rule whose selected fingerprint +style fails emits a debug log with the rule ID and rejected path. Glob patterns are case-sensitive. `*` matches within a single path component, while `**` matches recursively: `/assets/*.js` matches `/assets/app.js` but not @@ -1107,8 +1103,8 @@ edge_ttl_seconds = 31536000 immutable = true ``` -**Publisher allowlist example** (enable only after verifying the filename -convention): +**Publisher allowlist example** (enable only for an unambiguous immutable +filename convention): ```toml [[cache.asset_rules]] @@ -1120,7 +1116,7 @@ path_globs = [ "/assets/**/*.png", "/assets/**/*.webp", ] -fingerprint_style = "vite-base64-url" +fingerprint_style = "hex" visibility = "public" browser_ttl_seconds = 31536000 edge_ttl_seconds = 31536000 diff --git a/trusted-server.example.toml b/trusted-server.example.toml index ed4afa1cb..bfc1c6c6e 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -133,9 +133,10 @@ enabled = false # id = "publisher-fingerprinted-assets" # enabled = false # path_globs = ["/assets/**/*.js", "/assets/**/*.css", "/assets/**/*.png", "/assets/**/*.webp"] -# Immutable custom rules require an explicit fingerprint_style selected for the -# publisher's bundler, for example "hex", "esbuild-base32", or "vite-base64-url". -# fingerprint_style = "vite-base64-url" +# Immutable custom rules require an unambiguous fingerprint_style, either +# "hex" or "esbuild-base32". "vite-base64-url" is allowed only for +# non-immutable rules because ordinary filenames can match its shape. +# fingerprint_style = "hex" # visibility = "public" # browser_ttl_seconds = 31536000 # edge_ttl_seconds = 31536000 From 49e7094a7bfb25173fdf9f6bf20b4ebb94105f2b Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 20 Aug 2026 09:02:41 -0500 Subject: [PATCH 392/395] Address GPT diagnostics review feedback --- .../lib/src/integrations/gpt/index.ts | 3 +- .../integrations/gpt_diagnostics/badges.ts | 25 ++-- .../integrations/gpt_diagnostics/binding.ts | 12 +- .../integrations/gpt_diagnostics/overlay.ts | 18 +-- .../gpt_diagnostics/presentation_helpers.ts | 18 +++ .../gpt_diagnostics/slot_size_observer.ts | 19 ++- .../src/integrations/gpt_diagnostics/store.ts | 4 +- .../lib/test/integrations/gpt/ad_init.test.ts | 19 ++- .../gpt_diagnostics/badges.test.ts | 14 ++ .../gpt_diagnostics/overlay.test.ts | 4 +- .../slot_size_observer.test.ts | 121 ++++++++++++++++-- .../gpt_diagnostics/types.test.ts | 5 + docs/guide/integrations/gpt-diagnostics.md | 12 +- 13 files changed, 206 insertions(+), 68 deletions(-) create mode 100644 crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/presentation_helpers.ts diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 88bf659fe..b339c99ed 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1049,6 +1049,7 @@ export function installTsAdInit(): void { if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); }); gptSlot.setTargeting(TS_INITIAL_TARGETING_KEY, '1'); + const requestedSlotSizes = ts.gptSlotHandoffs?.[slotDivId2]?.formats; // Diagnostics are observational only. A missing or malformed debug // implementation must never interrupt slot mapping or delivery. try { @@ -1058,7 +1059,7 @@ export function installTsAdInit(): void { slot.id, opportunity, bid.hb_auction_id, - slot.formats + requestedSlotSizes ); } catch { // Diagnostics must not alter ad delivery. diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts index 408fcb1f9..4dc1250a2 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts @@ -2,6 +2,7 @@ import type { GptDiagnosticsRequestCycle } from '../../core/types'; import type { GptDiagnosticsBindingManager } from './binding'; import { unhandledCase } from './exhaustive'; +import { formatSizes, scheduleFrame } from './presentation_helpers'; import type { GptDiagnosticsBindingInput, GptDiagnosticsStoreSlotSnapshot, @@ -26,6 +27,7 @@ type BadgeWindow = Window & { const BADGE_MAX_WIDTH_PX = 260; const BADGE_EDGE_GUTTER_PX = 4; +const MAX_BADGE_REQUESTED_SLOT_SIZES = 3; interface BadgeOptions { window?: BadgeWindow; @@ -33,14 +35,6 @@ interface BadgeOptions { scheduleFrame?: (callback: () => void) => void; } -function defaultScheduleFrame(callback: () => void): void { - if (typeof requestAnimationFrame === 'function') { - requestAnimationFrame(() => callback()); - } else { - queueMicrotask(callback); - } -} - function intersectsViewport(rectangle: DOMRect, window: Window): boolean { return ( rectangle.width > 0 && @@ -106,10 +100,6 @@ function deliveryLabel(cycle: GptDiagnosticsRequestCycle): string | undefined { } } -function formatSizes(sizes: ReadonlyArray): string { - return sizes.map((size) => `${size[0]}×${size[1]}`).join(', '); -} - function badgeText(cycle: GptDiagnosticsRequestCycle): string { const firstLine: string[] = []; if (cycle.isEmpty === true) firstLine.push('Empty'); @@ -120,7 +110,13 @@ function badgeText(cycle: GptDiagnosticsRequestCycle): string { if (delivery) firstLine.push(delivery); if (cycle.requestPath === 'competing') firstLine.push('Competing paths'); if (cycle.requestedSlotSizes) { - firstLine.push(`Requested ${formatSizes(cycle.requestedSlotSizes)}`); + const displayedSizes = cycle.requestedSlotSizes.slice(0, MAX_BADGE_REQUESTED_SLOT_SIZES); + const remainingSizeCount = cycle.requestedSlotSizes.length - displayedSizes.length; + firstLine.push( + `Requested ${formatSizes(displayedSizes)}${ + remainingSizeCount > 0 ? ` +${remainingSizeCount}` : '' + }` + ); } if (cycle.size) firstLine.push(`GPT fill ${cycle.size[0]}×${cycle.size[1]}`); if (cycle.observedSlotSize) { @@ -169,7 +165,8 @@ export class GptDiagnosticsBadgeManager { this.bindings = bindings; this.window = options.window ?? (window as unknown as BadgeWindow); this.document = options.document ?? document; - this.scheduleFrame = options.scheduleFrame ?? defaultScheduleFrame; + this.scheduleFrame = + options.scheduleFrame ?? ((callback) => scheduleFrame(this.window, callback)); this.refreshSlotElementIds(); this.unsubscribeStore = this.store.subscribe(() => { this.refreshSlotElementIds(); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/binding.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/binding.ts index 489e7beb1..c6c46a727 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/binding.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/binding.ts @@ -1,5 +1,6 @@ import type { GptDiagnosticsBinding, GptDiagnosticsSlotExport } from '../../core/types'; +import { scheduleFrame } from './presentation_helpers'; import type { GptDiagnosticsBindingInput } from './store'; interface BindingStore { @@ -27,14 +28,6 @@ export interface GptDiagnosticsBindingView { type BindingListener = () => void; -function defaultScheduleFrame(callback: () => void): void { - if (typeof requestAnimationFrame === 'function') { - requestAnimationFrame(() => callback()); - } else { - queueMicrotask(callback); - } -} - function isVisibleInViewport(element: HTMLElement, window: BindingWindow): boolean { const rectangle = element.getBoundingClientRect(); if (rectangle.width <= 0 || rectangle.height <= 0) return false; @@ -97,7 +90,8 @@ export class GptDiagnosticsBindingManager { this.store = store; this.document = options.document ?? document; this.window = options.window ?? (window as unknown as BindingWindow); - this.scheduleFrame = options.scheduleFrame ?? defaultScheduleFrame; + this.scheduleFrame = + options.scheduleFrame ?? ((callback) => scheduleFrame(this.window, callback)); this.unsubscribeStore = this.store.subscribe(() => this.scheduleRefresh()); this.window.addEventListener('scroll', this.scheduleRefresh, { passive: true }); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts index 1eeb4976e..63c0b6fb3 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts @@ -3,6 +3,7 @@ import type { GptDiagnosticsRequestCycle } from '../../core/types'; import type { GptDiagnosticsBindingManager } from './binding'; import { unhandledCase } from './exhaustive'; +import { formatSizes, scheduleFrame } from './presentation_helpers'; import type { GptDiagnosticsStoreSlotSnapshot, GptDiagnosticsStoreSnapshot } from './store'; export const GPT_DIAGNOSTICS_HOST_ID = 'trusted-server-gpt-diagnostics'; @@ -99,14 +100,6 @@ const PANEL_STYLES = ` } `; -function defaultScheduleFrame(callback: () => void): void { - if (typeof requestAnimationFrame === 'function') { - requestAnimationFrame(() => callback()); - } else { - queueMicrotask(callback); - } -} - function latestCycle( slot: GptDiagnosticsStoreSlotSnapshot ): GptDiagnosticsRequestCycle | undefined { @@ -278,11 +271,7 @@ function cycleFacts(cycle: GptDiagnosticsRequestCycle): string[] { if (cycle.viewableAtMs !== undefined) facts.push('GPT impressionViewable observed'); if (cycle.incompleteSequence) facts.push('Incomplete sequence'); if (cycle.requestedSlotSizes) { - facts.push( - `Requested slot sizes ${cycle.requestedSlotSizes - .map((size) => `${size[0]}×${size[1]}`) - .join(', ')}` - ); + facts.push(`Requested slot sizes ${formatSizes(cycle.requestedSlotSizes)}`); } if (cycle.size) facts.push(`GPT-reported fill size ${cycle.size[0]}×${cycle.size[1]}`); if (cycle.observedSlotSize) { @@ -368,7 +357,8 @@ export class GptDiagnosticsOverlay { this.bindings = bindings; this.window = options.window ?? (window as unknown as OverlayWindow); this.document = options.document ?? document; - this.scheduleFrame = options.scheduleFrame ?? defaultScheduleFrame; + this.scheduleFrame = + options.scheduleFrame ?? ((callback) => scheduleFrame(this.window, callback)); this.onExport = options.onExport ?? (() => undefined); this.onShadowRoot = options.onShadowRoot; this.onBadgeLayerChange = options.onBadgeLayerChange; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/presentation_helpers.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/presentation_helpers.ts new file mode 100644 index 000000000..b601833b8 --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/presentation_helpers.ts @@ -0,0 +1,18 @@ +import type { Size } from '../../core/types'; + +/** Formats CSS sizes consistently across diagnostics presentation surfaces. */ +export function formatSizes(sizes: ReadonlyArray): string { + return sizes.map((size) => `${size[0]}×${size[1]}`).join(', '); +} + +/** Schedules presentation work in the target window's next animation frame. */ +export function scheduleFrame( + window: Pick, + callback: () => void +): void { + if (typeof window.requestAnimationFrame === 'function') { + window.requestAnimationFrame(() => callback()); + } else { + queueMicrotask(callback); + } +} diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts index ab49a451a..c3328f992 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts @@ -1,6 +1,7 @@ import type { Size } from '../../core/types'; import type { GptDiagnosticsBindingManager } from './binding'; +import { scheduleFrame } from './presentation_helpers'; import type { GptDiagnosticsStoreSnapshot } from './store'; interface SlotSizeStore { @@ -15,6 +16,7 @@ interface SlotSizeBindings { } type SlotSizeWindow = Window & { + HTMLElement: typeof HTMLElement; ResizeObserver?: typeof ResizeObserver; }; @@ -28,14 +30,6 @@ interface ObservedCycle { requestNumber: number; } -function defaultScheduleFrame(callback: () => void): void { - if (typeof requestAnimationFrame === 'function') { - requestAnimationFrame(() => callback()); - } else { - queueMicrotask(callback); - } -} - function latestFilledCycle( slot: GptDiagnosticsStoreSnapshot['slots'][number] ): ObservedCycle | undefined { @@ -70,7 +64,8 @@ export class GptDiagnosticsSlotSizeObserver { this.store = store; this.bindings = bindings; this.window = options.window ?? (window as unknown as SlotSizeWindow); - this.scheduleFrame = options.scheduleFrame ?? defaultScheduleFrame; + this.scheduleFrame = + options.scheduleFrame ?? ((callback) => scheduleFrame(this.window, callback)); this.unsubscribeStore = this.store.subscribe(this.scheduleRefresh); this.unsubscribeBindings = this.bindings.subscribe(this.scheduleRefresh); this.refresh(); @@ -124,6 +119,8 @@ export class GptDiagnosticsSlotSizeObserver { } private measure(element: HTMLElement, cycle: ObservedCycle): void { + if (this.destroyed) return; + const binding = this.bindings.get(cycle.runtimeSlotNumber); if (binding.binding.status !== 'bound' || binding.element !== element || !element.isConnected) { return; @@ -139,8 +136,8 @@ export class GptDiagnosticsSlotSizeObserver { return; } this.store.recordObservedSlotSize(cycle.runtimeSlotNumber, cycle.requestNumber, [ - rectangle.width, - rectangle.height, + Math.round(rectangle.width), + Math.round(rectangle.height), ]); } } diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts index 03d887aa0..ca3348ae9 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts @@ -716,7 +716,9 @@ export class GptDiagnosticsStore { } const record = this.slots.get(runtimeSlotNumber); - const cycle = record?.requests.find((candidate) => candidate.requestNumber === requestNumber); + if (!record) return; + + const cycle = record.requests.find((candidate) => candidate.requestNumber === requestNumber); if ( !cycle || record.requests[record.requests.length - 1] !== cycle || diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index ddeecb315..2a56cd606 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -296,7 +296,7 @@ describe('installTsAdInit', () => { 'atf_sidebar_ad', expectedOpportunity, undefined, - [[300, 250]] + undefined ); } ); @@ -322,11 +322,11 @@ describe('installTsAdInit', () => { 'atf_sidebar_ad', 'unrenderable_candidate', 'auction-123', - [[300, 250]] + undefined ); }); - it('captures every configured Trusted Server format when associating a GPT slot', async () => { + it('retains handoff formats when reusing a Trusted Server-defined GPT slot', async () => { const recordTrustedServerOpportunity = vi.fn(); const formats: Array<[number, number]> = [ [300, 250], @@ -338,6 +338,17 @@ describe('installTsAdInit', () => { recordTrustedServerOpportunity, formats ); + (window as TestWindow).tsjs!.gptSlotHandoffs = { + 'div-atf-sidebar': { + gamUnitPath: '/123/atf', + formats, + divIdPrefix: 'div-atf-sidebar', + slotElementId: 'div-atf-sidebar', + publisherClaimed: true, + suppressPublisherDisplay: false, + suppressPublisherRefresh: false, + }, + }; const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); @@ -366,7 +377,7 @@ describe('installTsAdInit', () => { 'atf_sidebar_ad', 'no_candidate', undefined, - [[300, 250]] + undefined ); }); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts index 7ac981b6d..80c240a3e 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts @@ -268,6 +268,20 @@ describe('GptDiagnosticsBadgeManager', () => { ).toBe( 'Filled · Requested 728×90, 970×250 · GPT fill 728×90 · Outer box 980×270\nResponse 276 ms · Render 42 ms\nViewable after 1 s' ); + expect( + gptDiagnosticsBadgeTextForTest({ + requestNumber: 1, + isEmpty: false, + requestedSlotSizes: [ + [300, 250], + [320, 50], + [728, 90], + [970, 250], + ], + incompleteSequence: false, + durations: {}, + }) + ).toBe('Filled · Requested 300×250, 320×50, 728×90 +1'); expect( gptDiagnosticsBadgeTextForTest({ requestNumber: 1, diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts index 9c9765ed1..41cad667a 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts @@ -413,6 +413,8 @@ describe('GptDiagnosticsOverlay', () => { [ [300, 250], [728, 90], + [320, 50], + [970, 250], ] ); store.recordSlotRequested(filledSlot); @@ -468,7 +470,7 @@ describe('GptDiagnosticsOverlay', () => { expect(root!.textContent).toContain('/example/site/filled-slot'); expect(root!.textContent).toContain('Empty'); expect(root!.textContent).toContain('Previous requests (1)'); - expect(root!.textContent).toContain('Requested slot sizes 300×250, 728×90'); + expect(root!.textContent).toContain('Requested slot sizes 300×250, 728×90, 320×50, 970×250'); expect(root!.textContent).toContain('GPT-reported fill size 300×250'); expect(root!.textContent).toContain('Observed outer slot box 320×270'); expect(root!.textContent).toContain('Backfill yes'); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts index 86ad532c7..3709221ab 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts @@ -4,6 +4,11 @@ import type { GptDiagnosticsRequestCycle } from '../../../src/core/types'; import { GptDiagnosticsSlotSizeObserver } from '../../../src/integrations/gpt_diagnostics/slot_size_observer'; import type { GptDiagnosticsStoreSnapshot } from '../../../src/integrations/gpt_diagnostics/store'; +type SlotSizeTestWindow = Window & { + HTMLElement: typeof HTMLElement; + ResizeObserver?: typeof ResizeObserver; +}; + class ResizeObserverMock { static instances: ResizeObserverMock[] = []; readonly observe = vi.fn(); @@ -67,7 +72,7 @@ describe('GptDiagnosticsSlotSizeObserver', () => { const element = document.createElement('div'); document.body.append(element); const getBoundingClientRect = vi.spyOn(element, 'getBoundingClientRect'); - getBoundingClientRect.mockReturnValue({ width: 728, height: 90 } as DOMRect); + getBoundingClientRect.mockReturnValue({ width: 728.4, height: 90.5 } as DOMRect); const requests = [cycle(1, false)]; requests[0].size = [1, 1]; const store = { @@ -81,19 +86,111 @@ describe('GptDiagnosticsSlotSizeObserver', () => { }; const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { - window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as SlotSizeTestWindow, scheduleFrame: (callback) => callback(), }); - expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 1, [728, 90]); + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 1, [728, 91]); expect(requests[0].size).toEqual([1, 1]); - getBoundingClientRect.mockReturnValue({ width: 970, height: 250 } as DOMRect); - ResizeObserverMock.instances.at(-1)!.emit(element); + getBoundingClientRect.mockReturnValue({ width: 969.6, height: 250.2 } as DOMRect); + ResizeObserverMock.instances[ResizeObserverMock.instances.length - 1]!.emit(element); expect(store.recordObservedSlotSize).toHaveBeenLastCalledWith(1, 1, [970, 250]); observer.destroy(); }); + it('uses the provided window to schedule the initial measurement', () => { + const element = document.createElement('div'); + document.body.append(element); + vi.spyOn(element, 'getBoundingClientRect').mockReturnValue({ + width: 300, + height: 250, + } as DOMRect); + const requestAnimationFrame = vi.fn((callback: FrameRequestCallback) => { + callback(0); + return 1; + }); + const store = { + snapshot: () => snapshot([cycle(1, false)]), + recordObservedSlotSize: vi.fn(), + subscribe: () => () => undefined, + }; + const bindings = { + get: () => ({ binding: { status: 'bound' as const }, element, visible: true }), + subscribe: () => () => undefined, + }; + + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { + HTMLElement, + ResizeObserver: ResizeObserverMock, + requestAnimationFrame, + } as unknown as SlotSizeTestWindow, + }); + + expect(requestAnimationFrame).toHaveBeenCalledTimes(1); + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 1, [300, 250]); + observer.destroy(); + }); + + it('records one initial measurement when ResizeObserver is unavailable', () => { + const element = document.createElement('div'); + document.body.append(element); + const getBoundingClientRect = vi.spyOn(element, 'getBoundingClientRect'); + getBoundingClientRect.mockReturnValue({ width: 300, height: 250 } as DOMRect); + const store = { + snapshot: () => snapshot([cycle(1, false)]), + recordObservedSlotSize: vi.fn(), + subscribe: () => () => undefined, + }; + const bindings = { + get: () => ({ binding: { status: 'bound' as const }, element, visible: true }), + subscribe: () => () => undefined, + }; + + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { HTMLElement } as unknown as SlotSizeTestWindow, + scheduleFrame: (callback) => callback(), + }); + + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 1, [300, 250]); + expect(ResizeObserverMock.instances).toHaveLength(0); + getBoundingClientRect.mockReturnValue({ width: 728, height: 90 } as DOMRect); + expect(store.recordObservedSlotSize).toHaveBeenCalledTimes(1); + observer.destroy(); + }); + + it('does not measure after destruction when a frame is pending', () => { + const element = document.createElement('div'); + document.body.append(element); + vi.spyOn(element, 'getBoundingClientRect').mockReturnValue({ + width: 300, + height: 250, + } as DOMRect); + const frames: Array<() => void> = []; + const store = { + snapshot: () => snapshot([cycle(1, false)]), + recordObservedSlotSize: vi.fn(), + subscribe: () => () => undefined, + }; + const bindings = { + get: () => ({ binding: { status: 'bound' as const }, element, visible: true }), + subscribe: () => () => undefined, + }; + + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { + HTMLElement, + ResizeObserver: ResizeObserverMock, + } as unknown as SlotSizeTestWindow, + scheduleFrame: (callback) => frames.push(callback), + }); + observer.destroy(); + frames.shift()!(); + + expect(store.recordObservedSlotSize).not.toHaveBeenCalled(); + }); + it.each(['unbound', 'ambiguous'] as const)('does not observe %s slots', (status) => { const element = document.createElement('div'); document.body.append(element); @@ -108,12 +205,17 @@ describe('GptDiagnosticsSlotSizeObserver', () => { }; const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { - window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + window: { + HTMLElement, + ResizeObserver: ResizeObserverMock, + } as unknown as SlotSizeTestWindow, scheduleFrame: (callback) => callback(), }); expect(store.recordObservedSlotSize).not.toHaveBeenCalled(); - expect(ResizeObserverMock.instances.at(-1)!.observe).not.toHaveBeenCalled(); + expect( + ResizeObserverMock.instances[ResizeObserverMock.instances.length - 1]!.observe + ).not.toHaveBeenCalled(); observer.destroy(); }); @@ -140,7 +242,10 @@ describe('GptDiagnosticsSlotSizeObserver', () => { }; const frames: Array<() => void> = []; const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { - window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + window: { + HTMLElement, + ResizeObserver: ResizeObserverMock, + } as unknown as SlotSizeTestWindow, scheduleFrame: (callback) => frames.push(callback), }); const firstObserver = ResizeObserverMock.instances[0]; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/types.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/types.test.ts index 6b3103060..f4f4c7486 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/types.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/types.test.ts @@ -11,6 +11,7 @@ import type { GptDiagnosticsRequestPath, GptDiagnosticsResponseClass, GptDiagnosticsSlotExport, + Size, TsjsApi, } from '../../../src/core/types'; @@ -138,6 +139,8 @@ describe('GPT diagnostics public types', () => { requestPath: 'publisher_refresh', requestIntentId: 7, trustedServerAuctionId: 'ts-auc-example', + requestedSlotSizes: [[300, 250]], + observedSlotSize: [728, 90], opportunityToRequestMs: 24, replacedRequestNumber: 1, previousRenderToRequestMs: 6048, @@ -174,6 +177,8 @@ describe('GPT diagnostics public types', () => { expectTypeOf(evidenceCycle.requestPath).toEqualTypeOf(); expectTypeOf(evidenceCycle.requestIntentId).toEqualTypeOf(); expectTypeOf(evidenceCycle.trustedServerAuctionId).toEqualTypeOf(); + expectTypeOf(evidenceCycle.requestedSlotSizes).toEqualTypeOf | undefined>(); + expectTypeOf(evidenceCycle.observedSlotSize).toEqualTypeOf(); expectTypeOf(evidenceSnapshot.attributionIssues).toEqualTypeOf< GptDiagnosticsAttributionIssue[] | undefined >(); diff --git a/docs/guide/integrations/gpt-diagnostics.md b/docs/guide/integrations/gpt-diagnostics.md index 8c7c00f18..7e677938a 100644 --- a/docs/guide/integrations/gpt-diagnostics.md +++ b/docs/guide/integrations/gpt-diagnostics.md @@ -293,12 +293,14 @@ separate facts as requested slot sizes, GPT-reported fill size, and observed out box. The observed box may differ from GPT's reported size (for example, a flexible APS creative can report `1×1` while its allocated outer slot box is larger). It is a publisher-page layout measurement, not a claim about universal internal creative-pixel -dimensions. Empty, unbound, missing, or ambiguous slots do not report an observed box; -delayed measurements from an older cycle are rejected after a refresh. +dimensions. A collapsed or hidden bound element can report `0×0`, which records the +page layout state rather than an invalid measurement. Empty, unbound, missing, or +ambiguous slots do not report an observed box; delayed measurements from an older cycle +are rejected after a refresh. Cross-origin and SafeFrame boundaries prevent diagnostics from inspecting iframe -content. It does not inspect iframe content or alter the APS sandbox, so it cannot -use this field to prove the inner creative's pixels. +content or altering the APS sandbox, so this field cannot prove the inner creative's +pixels. Badges and the panel live in a closed Shadow DOM. Diagnostics do not add attributes, classes, or inline styles to publisher slot elements. @@ -388,7 +390,7 @@ inaccessible to JavaScript. - Retained request cycles per slot: 10. - Retained callback issues: 128. - Retained auction-slot-to-GPT-slot associations: 64. -- Requested slot sizes per correlated request: 16 valid positive sizes. +- Requested slot sizes per correlated request: the first 16 configured entries, with invalid entries dropped. - Retained creative attempts, including status tombstones: 128. - Retained attribution issues: 128. From 6c416e65897a1e909960a662f4724c5e32227681 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 20 Aug 2026 11:09:24 -0500 Subject: [PATCH 393/395] Format configuration guide --- docs/guide/configuration.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index a210df94b..44f038f96 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1505,6 +1505,7 @@ URL. Error responses and non-document requests retain the origin policy. Any response that carries `Set-Cookie` is finalized as `Cache-Control: private, max-age=0`; this privacy rule takes precedence over the short inactive-stack policy. + ```toml [creative_opportunities] enabled = true # set to false to disable server-side ad templates @@ -1531,6 +1532,7 @@ Because EdgeZero only replaces TOML leaves that already exist, first add `enabled = true` to the `[creative_opportunities]` block in the base config before using this override. See [Environment Variable Overrides (Typed CLI)](#environment-variable-overrides-typed-cli) for the general overlay rules. + ```bash TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false ``` From 2234e9bafb114b9f69804dac359e3b53a5bb5ed6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 20 Aug 2026 22:10:01 +0530 Subject: [PATCH 394/395] Remove legacy C2 harness from release CI --- .github/workflows/test.yml | 5 - scripts/c2-local-test.sh | 607 ------------------------------------- 2 files changed, 612 deletions(-) delete mode 100755 scripts/c2-local-test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a3fc03b00..58e6a4ac7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,11 +58,6 @@ jobs: - name: Run tests run: cargo test-fastly - - name: Run C2 ESI local harness - run: ./scripts/c2-local-test.sh esi - - - name: Run inline control harness - run: ./scripts/c2-local-test.sh inline - name: Run template cache ESI local harness run: BID_DELAY=3 ./scripts/template-cache-local-test.sh esi diff --git a/scripts/c2-local-test.sh b/scripts/c2-local-test.sh deleted file mode 100755 index cd987551a..000000000 --- a/scripts/c2-local-test.sh +++ /dev/null @@ -1,607 +0,0 @@ -#!/usr/bin/env bash -# -# Local harness for the #1009 shared-template cache (C2). -# -# Runs Trusted Server under Viceroy against a stub origin and asserts the cache -# behaves. Everything it needs is generated into a temp directory; your -# `trusted-server.toml` is never read and your tracked `fastly.toml` is never -# modified, including while the harness is running. -# -# Usage: -# ./scripts/c2-local-test.sh # esi mode (shared template + edge assembly) -# ./scripts/c2-local-test.sh inline # today's shipped behaviour, as a control -# -# Spike-only. Remove with the spike. - -set -euo pipefail - -MODE="${1:-esi}" -case "$MODE" in - inline | esi) ;; - *) - echo "Unknown mode '$MODE'. Use one of: inline, esi." >&2 - exit 1 - ;; -esac -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -WORK="$(mktemp -d)" -ORIGIN_PORT="${ORIGIN_PORT:-9099}" -TS_PORT="${TS_PORT:-7788}" -HOST_TRIPLE="$(rustc -vV | sed -n 's/^host: //p')" - -# The stub's bid endpoint sleeps this long. It exists to make the auction -# observable in the timings: with an instant auction, buffered and streaming -# assembly are indistinguishable. -BID_DELAY="${BID_DELAY:-1.5}" - -PASS=0 -FAIL=0 - -info() { printf '\n\033[1m==> %s\033[0m\n' "$*"; } -ok() { printf ' \033[32mPASS\033[0m %s\n' "$*"; PASS=$((PASS + 1)); } -bad() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; FAIL=$((FAIL + 1)); } - -check() { # check - if [ "$2" = "$3" ]; then ok "$1 ($2)"; else bad "$1 — got '$2', want '$3'"; fi -} - -cleanup() { - local status=$? - [ -n "${VICEROY_PID:-}" ] && kill "$VICEROY_PID" 2>/dev/null || true - [ -n "${ORIGIN_PID:-}" ] && kill "$ORIGIN_PID" 2>/dev/null || true - rm -rf "$WORK" - exit $status -} -trap cleanup EXIT INT TERM - -command -v viceroy >/dev/null || { - echo "viceroy not found. Install: cargo install viceroy --version 0.17.0 --locked" >&2 - exit 1 -} -command -v node >/dev/null || { - echo "node not found. The harness executes the real GPT bundle to verify slot setup." >&2 - exit 1 -} - -# A port already in use means requests would go to something else entirely — most -# likely a leftover run, whose warm cache and stale config would read as a result. -for port in "$ORIGIN_PORT" "$TS_PORT"; do - if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then - echo "Port $port is already in use. Stop the process, or set" >&2 - echo "ORIGIN_PORT / TS_PORT to something free." >&2 - lsof -nP -iTCP:"$port" -sTCP:LISTEN >&2 - exit 1 - fi -done - -info "Building (debug wasm + ts CLI)" -cargo build --package trusted-server-adapter-fastly --target wasm32-wasip1 >/dev/null -cargo build -p trusted-server-cli --target "$HOST_TRIPLE" >/dev/null -WASM="$REPO_ROOT/target/wasm32-wasip1/debug/trusted-server-adapter-fastly.wasm" -TS="$REPO_ROOT/target/$HOST_TRIPLE/debug/ts" - -info "Starting stub origin on :$ORIGIN_PORT" -cat > "$WORK/origin.py" <stub creative", - "w": 728, - "h": 90, - } - ], - } - ], -} - -PAGE = b""" -Stub article - -

Stub article

-
-

Body copy.

- -""" - -class H(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def _send(self, body, ctype, extra=()): - self.send_response(200) - self.send_header("Content-Type", ctype) - for k, v in extra: - self.send_header(k, v) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def do_GET(self): - print(f"origin: received GET {self.path}", flush=True) - # Compresses when asked, because a real origin does and because a plaintext-only - # stub hid a bug that broke the feature end to end: a gzip template has no - # findable seam marker, and splicing plaintext bids into a gzip stream gives the - # browser ERR_CONTENT_DECODING_FAILED. - base = [ - ("Cache-Control", "public, max-age=60"), - ("Surrogate-Control", "max-age=1200, stale-while-revalidate=21600, stale-if-error=604800"), - ("Vary", "Accept-Encoding"), - ] - if "gzip" in (self.headers.get("Accept-Encoding") or ""): - print("origin: served COMPRESSED", flush=True) - self._send(gzip.compress(PAGE), "text/html; charset=utf-8", - base + [("Content-Encoding", "gzip")]) - else: - print("origin: served PLAINTEXT", flush=True) - self._send(PAGE, "text/html; charset=utf-8", base) - - def do_POST(self): - print(f"origin: received POST {self.path}", flush=True) - n = int(self.headers.get("Content-Length") or 0) - if n: - self.rfile.read(n) - time.sleep($BID_DELAY) - self._send(json.dumps(BID_RESPONSE).encode(), "application/json") - - def log_message(self, fmt, *args): - print("origin: " + fmt % args, flush=True) - -ThreadingHTTPServer(("127.0.0.1", $ORIGIN_PORT), H).serve_forever() -PYEOF -python3 "$WORK/origin.py" > "$WORK/origin.log" 2>&1 & -ORIGIN_PID=$! -sleep 1 - -info "Generating stub config (mode: $MODE)" -python3 - "$REPO_ROOT/trusted-server.example.toml" "$WORK/app.toml" "$MODE" "$ORIGIN_PORT" <<'PYEOF' -import sys, re -src, out, mode, port = sys.argv[1:5] -s = open(src).read() - -s = s.replace('origin_url = "https://origin.example.com"', f'origin_url = "http://127.0.0.1:{port}"', 1) -# The example config ships placeholders that validation rejects outright. -s = s.replace('password = "replace-with-admin-password-32-bytes"', - 'password = "local-harness-admin-password-not-a-real-one"', 1) -s = s.replace('proxy_secret = "change-me-proxy-secret"', - 'proxy_secret = "local-harness-proxy-secret-not-a-real-one"', 1) -s = re.sub(r'passphrase = "[^"]*"', - 'passphrase = "local-harness-ec-passphrase-not-a-real-one"', s, count=1) - -# A real auction, pointed at the stub's slow endpoint, so the timings mean something. -s = s.replace('[integrations.prebid]\nenabled = false\nserver_url = "https://prebid.example.com/openrtb2/auction"', - f'[integrations.prebid]\nenabled = true\nserver_url = "http://127.0.0.1:{port}/bid"\n' - 'external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-stub.js"', 1) -s = s.replace('providers = []', 'providers = ["prebid"]', 1) -s = s.replace('\n[proxy]\n', '\n[proxy]\nallowed_domains = ["assets.example.com", "127.0.0.1"]\n', 1) -s = s.replace('[auction]\nenabled = false', '[auction]\nenabled = true', 1) -s = s.replace('auction_timeout_ms = 500', 'auction_timeout_ms = 3000', 1) -s = s.replace('timeout_ms = 2000', 'timeout_ms = 3000', 1) - -# The spike keys go directly under the table header. The slot is a table of its own -# and must go at the end: inserted here it would swallow every scalar key that -# follows into `[[creative_opportunities.slot]]`. -scalars = f'''assembly_mode = "{mode}" -template_cache_vary = [] -origin_is_cookie_independent = true''' -lines = s.split("\n") -lines.insert(lines.index("[creative_opportunities]") + 1, scalars) -lines.append(''' -[[creative_opportunities.slot]] -id = "ts-slot-header" -div_id = "ts-slot-header" -page_patterns = ["/article"] -formats = [{ width = 728, height = 90 }] -''') -open(out, "w").write("\n".join(lines)) -PYEOF - -"$TS" config validate --app-config "$WORK/app.toml" >/dev/null - -info "Seeding an isolated config store (tracked fastly.toml remains untouched)" -# `config push --local` edits the adapter manifest. Give it a complete temporary -# project instead of editing the tracked manifest and trying to restore it afterward: -# a SIGKILL, machine crash, or failed restore could otherwise leave real serialized -# credentials in a tracked file. The crates symlink keeps manifest path validation -# pointed at this checkout without copying the workspace. -cp "$REPO_ROOT/edgezero.toml" "$WORK/edgezero.toml" -cp "$REPO_ROOT/fastly.toml" "$WORK/fastly.toml" -ln -s "$REPO_ROOT/crates" "$WORK/crates" -(cd "$WORK" && "$TS" config push --adapter fastly --local \ - --manifest "$WORK/edgezero.toml" --app-config "$WORK/app.toml" \ - --no-diff --yes >/dev/null) - -info "Starting Trusted Server on :$TS_PORT" -# Deliberately not wrapped in a subshell: `$!` would then be the subshell's pid, so -# cleanup would kill the wrapper and orphan viceroy. The next run would fail to bind -# and answer from the stale server instead — with the previous mode's config and a warm -# cache, which looks like a test result rather than a mistake. -RUST_LOG=info viceroy serve -C "$WORK/fastly.toml" \ - --addr "127.0.0.1:$TS_PORT" "$WASM" > "$WORK/viceroy.log" 2>&1 & -VICEROY_PID=$! - -for _ in $(seq 1 40); do - grep -q "Listening on" "$WORK/viceroy.log" 2>/dev/null && break - sleep 0.5 -done -if ! grep -q "Listening on" "$WORK/viceroy.log" 2>/dev/null; then - echo "Trusted Server failed to start. Last log lines:" >&2 - tail -20 "$WORK/viceroy.log" >&2 - exit 1 -fi - -req() { # req [extra curl args...] - local out="$1"; shift - curl -sS -D "$out.headers" -o "$out" \ - -w '%{time_starttransfer} %{time_total} %{http_code}' \ - -H "Host: ts.example.com" \ - -H "Accept-Encoding: gzip" \ - -H "sec-fetch-dest: document" -H "sec-fetch-mode: navigate" \ - "$@" "http://127.0.0.1:$TS_PORT/article" -} - -origin_gets() { grep -cF "origin: received GET /article" "$WORK/origin.log" || true; } - -info "Running assertions (mode: $MODE)" -BEFORE=$(origin_gets) -R1=$(req "$WORK/r1.html") -R2=$(req "$WORK/r2.html") - -if [ -s "$WORK/r1.html" ]; then - ok "first response has a body" -else - bad "first response body is empty" -fi -if [ -s "$WORK/r2.html" ]; then - ok "second response has a body" -else - bad "second response body is empty" -fi - -# Content assertions must never run against compressed bytes. `inline` responses stay -# gzipped end to end — only the shared path decodes, because its seam split is textual — -# and `grep` over a gzip stream matches nothing, which reads as a pass for every -# "must not contain" check and as a silent failure for every "must contain" one. -# Decode a copy and assert against that, in both modes. -SERVED="$WORK/r2.served.html" -if ! gzip -dc "$WORK/r2.html" > "$SERVED" 2>/dev/null; then - cp "$WORK/r2.html" "$SERVED" -fi -AFTER=$(origin_gets) -FETCHES=$((AFTER - BEFORE)) - -read -r TTFB1 TOTAL1 CODE1 <<< "$R1" -read -r TTFB2 TOTAL2 CODE2 <<< "$R2" - -for value in "$TTFB1" "$TOTAL1" "$CODE1" "$TTFB2" "$TOTAL2" "$CODE2"; do - if ! [[ "$value" =~ ^[0-9]+([.][0-9]+)?$ ]]; then - bad "curl returned a non-numeric timing/status field: '$value'" - fi -done - -check "first request returns 200" "$CODE1" "200" -check "second request returns 200" "$CODE2" "200" - -# The bid the stub origin returns, bucketed and then escaped the way the seam escapes -# it. Asserted in both modes: a shared-mode failure that inline shares would otherwise -# read as "the fixture never bids" rather than "the seam drops bids". -WINNING_BID='\"hb_pb\":\"4.25\"' -# Must stay in step with `AD_ASSEMBLY_SEAM` in publisher.rs. C2 stores this inert -# comment. The cold response turns it into a synthetic ESI include only in a private -# working copy; the warm response splits these bytes directly. -SEAM_MARKER='' - -c2_state() { - awk 'tolower($1) == "x-ts-c2-cache:" { gsub(/\r/, "", $2); print $2 }' "$1" | tail -1 -} - -assembly_state() { - awk 'tolower($1) == "x-ts-assembly:" { gsub(/\r/, "", $2); print $2 }' "$1" | tail -1 -} - -# Shared by the ESI assertions below. -check_hit_is_private() { - local hdrs - hdrs=$(curl -s -D- -o /dev/null -H "Host: ts.example.com" \ - -H "Accept-Encoding: gzip" \ - -H "sec-fetch-dest: document" -H "sec-fetch-mode: navigate" \ - "http://127.0.0.1:$TS_PORT/article") - check "cache hit is not shared-cacheable" \ - "$(echo "$hdrs" | grep -ci 'cache-control: private, no-store' || true)" "1" -} - -check_post_reaches_origin() { - local before - before=$(grep -cF "origin: received POST /article" "$WORK/origin.log" || true) - curl -s -o /dev/null -X POST -d 'x=1' -H "Host: ts.example.com" \ - -H "Accept-Encoding: gzip" \ - "http://127.0.0.1:$TS_PORT/article" - check "a POST still reaches the origin" \ - "$(( $(grep -cF "origin: received POST /article" "$WORK/origin.log" || true) - before ))" "1" -} - -if [ "$MODE" = "inline" ]; then - check "inline fetches the origin every time" "$FETCHES" "2" - check "inline writes no shared template" \ - "$(grep -c 'c2_template_cache stored' "$WORK/viceroy.log" || true)" "0" - check "inline delivers the winning bid" \ - "$(grep -cF "$WINNING_BID" "$SERVED" || true)" "1" -else - check "second request is served from cache" "$FETCHES" "1" - check "cold request reports a stored miss" "$(c2_state "$WORK/r1.html.headers")" "miss-stored" - check "warm request reports a cache hit" "$(c2_state "$WORK/r2.html.headers")" "hit" - check "cold response uses the repaired ESI parser" \ - "$(assembly_state "$WORK/r1.html.headers")" "esi-parser" - check "warm response keeps the streaming byte seam" \ - "$(assembly_state "$WORK/r2.html.headers")" "byte-seam" - check "no unresolved seam marker reaches the browser" \ - "$(grep -cF "$SEAM_MARKER" "$SERVED" || true)" "0" - check "a bids script is present" \ - "$(grep -c 'window.tsjs' "$SERVED" || true)" "1" - # `window.tsjs` alone passes while initial ads are dead: shared modes suppress the head - # slot script, so if the seam does not carry slots, `adSlots` stays `[]` and `adInit` - # defines nothing. This harness passed green through exactly that bug. - # The slots ride the scheduler call (`s(b,a)`) rather than a bare assignment, so the - # navigation-generation guard covers them; `var a=JSON.parse(...)` is where they land. - check "the seam carries slot definitions, not just bids" \ - "$(grep -c 'var a=JSON.parse' "$SERVED" || true)" "1" - check "the slot definitions reach the guarded scheduler" \ - "$(grep -cF 's(b,a)' "$SERVED" || true)" "1" - check "the slot definitions are populated, not an empty array" \ - "$(grep -c 'var a=JSON.parse("\[\]")' "$SERVED" || true)" "0" - # The assertion the harness was missing entirely. `window.tsjs` and populated slots - # both pass on a page whose bids are `{}` — which is what shared modes served, on - # every request, for as long as this file has existed. - check "the seam carries a real bid, not an empty map" \ - "$(grep -cF 'var b=JSON.parse("{}")' "$SERVED" || true)" "0" - check "the winning bid's bucketed price reaches the reader" \ - "$(grep -cF "$WINNING_BID" "$SERVED" || true)" "1" - - GPT_BUNDLE="" - while IFS= read -r -d '' candidate; do - if [ -z "$GPT_BUNDLE" ] || [ "$candidate" -nt "$GPT_BUNDLE" ]; then - GPT_BUNDLE="$candidate" - fi - done < <(find "$REPO_ROOT/target/wasm32-wasip1/debug/build" \ - -path '*/out/tsjs-gpt.js' -type f -print0) - if [ -z "$GPT_BUNDLE" ] || [ ! -s "$GPT_BUNDLE" ]; then - bad "the generated GPT module cannot be found" - else - cat > "$WORK/verify-seam.mjs" <<'NODEEOF' -import fs from "node:fs"; -import vm from "node:vm"; - -const [documentPath, gptPath] = process.argv.slice(2); -const html = fs.readFileSync(documentPath, "utf8"); -const gpt = fs.readFileSync(gptPath, "utf8"); -const scripts = [...html.matchAll(/