diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb0233b03..63008f356 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,6 +27,11 @@ jobs: run: echo "viceroy-version=$(grep '^viceroy ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT shell: bash + - name: Retrieve Node.js version + id: node-version + run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + shell: bash + - name: Set up Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: @@ -45,9 +50,20 @@ jobs: if: steps.cache-viceroy.outputs.cache-hit != 'true' run: cargo install viceroy --version "${{ steps.viceroy-version.outputs.viceroy-version }}" --locked --force + - name: Use Node.js for the served-seam contract + uses: actions/setup-node@v4 + with: + node-version: ${{ steps.node-version.outputs.node-version }} + - name: Run tests run: cargo test-fastly + - name: Run C2 ESI local harness + run: ./scripts/c2-local-test.sh esi + + - name: Run inline control harness + run: ./scripts/c2-local-test.sh inline + test-axum: name: cargo test (axum native) runs-on: ubuntu-latest @@ -215,8 +231,7 @@ jobs: cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings - name: cargo test - run: | - cargo test --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" + run: ./scripts/test-cli.sh test-typescript: name: vitest diff --git a/CHANGELOG.md b/CHANGELOG.md index c00487769..920b3c4df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking** — Replaced the legacy APS contextual integration with APS OpenRTB at `/e/pb/bid`. APS configuration now uses canonical `account_id` (`pub_id` remains a compatibility alias), no longer requires APS-specific slot IDs, and defaults script creative eligibility off. Operators must update the endpoint, disable native APS demand for Trusted Server cohorts, and prepare GAM/Universal Creative targeting for `hb_bidder=aps` before rollout. `aps` entries in Prebid bidder lists are logged and stripped. APS renderer winners now preserve the upstream bid `id`, omit `crid` when APS omits it, and carry `ext.trusted_server.renderer` instead of `adm`; external `/auction` consumers must support this response shape. - **Breaking** — All auction paths now forward only a validated publisher-owned page URL as `site.page`, removing query and fragment data. APS OpenRTB omits `site.ref`; the existing Prebid Server path continues to forward the browser `Referer` as `site.ref`. Query-driven sites may lose contextual targeting and per-page reporting signals that previously came from query parameters. +- Publisher HTML now uses `Cache-Control: max-age=60` when server-side ad templates are inactive, while preserving origin `private`/`no-store` policies and CDN-specific cache headers. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers. - **Breaking** — `bid_param_zone_overrides` inner values must now be JSON objects; previously non-object or empty values (`"header" = "x"`, `"header" = {}`) were accepted and silently produced a dead rule at runtime. They now fail at startup with a configuration error. Operators upgrading should audit their `bid_param_zone_overrides` config for non-object zone entries. - **Breaking** — Integration configuration strings are no longer globally reinterpreted as JSON scalars. Operators upgrading should audit `[integrations.*]` settings and use native TOML/typed-config booleans and numbers (for example, `enabled = true`, not `enabled = "true"`); quoted numeric and boolean scalars now fail validation instead of silently converting. - **Breaking** — Sourcepoint browser module inclusion now requires explicit `[integrations.sourcepoint].enabled = true`; operators relying on the previous unconditional Sourcepoint module should enable the integration before upgrading. diff --git a/Cargo.lock b/Cargo.lock index cb8f40c68..7a991915f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,7 +146,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom", + "nom 7.1.3", "num-traits", "rusticata-macros", "thiserror 1.0.69", @@ -254,6 +254,15 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -573,7 +582,18 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", ] [[package]] @@ -583,7 +603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -767,7 +787,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -916,6 +936,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1041,7 +1070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", @@ -1186,7 +1215,7 @@ checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" dependencies = [ "asn1-rs", "displaydoc", - "nom", + "nom 7.1.3", "num-bigint", "num-traits", "rusticata-macros", @@ -1398,7 +1427,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "toml", ] @@ -1406,7 +1435,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1434,7 +1463,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1449,7 +1478,7 @@ dependencies = [ "log", "serde_json", "tempfile", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", "worker", ] @@ -1457,7 +1486,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-stream", @@ -1479,14 +1508,14 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.18", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1506,14 +1535,14 @@ dependencies = [ "subtle", "thiserror 2.0.18", "toml", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "chrono", "clap", @@ -1538,7 +1567,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-compression", @@ -1569,14 +1598,14 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "log", "proc-macro2", "quote", "serde", "serde_json", - "syn 2.0.118", + "syn 3.0.3", "toml", "validator", ] @@ -1690,6 +1719,26 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "esi" +version = "0.7.1" +source = "git+https://github.com/stackpop/esi.git?rev=4c53feab4d22ad9a84641b4c46f3f63bc6d197e2#4c53feab4d22ad9a84641b4c46f3f63bc6d197e2" +dependencies = [ + "atoi", + "base64", + "bytes", + "chrono", + "fastly", + "html-escape", + "log", + "md5", + "nom 8.0.0", + "percent-encoding", + "rand 0.10.2", + "regex", + "thiserror 2.0.18", +] + [[package]] name = "etcetera" version = "0.10.0" @@ -2034,6 +2083,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -2188,6 +2238,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "html-escape" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9356095b4b41197bba32173600e1582792cda618f65d12f68e2e77d273413c5" + [[package]] name = "html5ever" version = "0.35.0" @@ -2914,6 +2970,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" +[[package]] +name = "md5" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" + [[package]] name = "memchr" version = "2.8.2" @@ -2984,6 +3046,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "num" version = "0.4.3" @@ -3477,7 +3548,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -3604,7 +3675,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -3624,7 +3695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -3637,7 +3708,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -3775,6 +3846,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20 0.10.1", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3813,6 +3895,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rcgen" version = "0.13.2" @@ -4079,7 +4167,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -4494,7 +4582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -4506,7 +4594,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.9.0", "opaque-debug", ] @@ -4518,7 +4606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -4769,6 +4857,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -5074,6 +5173,19 @@ dependencies = [ "winnow 0.7.15", ] +[[package]] +name = "toml_edit" +version = "0.25.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.3", +] + [[package]] name = "toml_parser" version = "1.1.2+spec-1.1.0" @@ -5273,9 +5385,11 @@ dependencies = [ "base64", "bytes", "chrono", + "derive_more", "edgezero-adapter-fastly", "edgezero-core", "error-stack", + "esi", "fastly", "fern", "futures", @@ -5322,6 +5436,7 @@ dependencies = [ "derive_more", "directories", "edgezero-cli", + "edgezero-core", "error-stack", "futures", "http-body-util", @@ -5335,12 +5450,13 @@ dependencies = [ "scraper", "serde", "serde_json", + "temp-env", "tempfile", "time", "tokio", "tokio-rustls", "toml", - "toml_edit", + "toml_edit 0.23.10+spec-1.0.0", "trusted-server-core", "url", "webpki-roots", @@ -5373,6 +5489,7 @@ dependencies = [ "hex", "hmac", "http", + "httpdate", "iab_gpp", "jose-jwk", "log", @@ -5896,7 +6013,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -6325,7 +6442,7 @@ dependencies = [ "data-encoding", "der-parser", "lazy_static", - "nom", + "nom 7.1.3", "oid-registry", "ring", "rusticata-macros", diff --git a/Cargo.toml b/Cargo.toml index 7ca87e687..ab5638f5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,12 +54,12 @@ criterion = { version = "0.5", default-features = false, features = ["cargo_benc derive_more = { version = "2.0", features = ["display", "error"] } directories = "5" ed25519-dalek = { version = "2.2", features = ["rand_core"] } -edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-cli = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4" } -edgezero-core = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } +edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-cli = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions" } +edgezero-core = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } env_logger = "0.11" error-stack = "0.6" fastly = "0.12" @@ -71,6 +71,7 @@ getrandom = "0.2" hex = "0.4.3" hmac = "0.12.1" http = "1.4.0" +httpdate = "1.0.3" http-body-util = "0.1" hyper = "1" hyper-util = "0.1" diff --git a/README.md b/README.md index b87fe61ad..81794720c 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ ts config init ts config validate # Audit a public page with Chrome/Chromium to bootstrap a draft config -ts audit https://publisher.example +ts audit generate https://publisher.example # Run tests (Fastly/WASM crates — requires Viceroy) cargo test-fastly diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 1bed830ac..897877870 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -11,7 +11,10 @@ use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; use trusted_server_core::proxy::{ @@ -183,7 +186,7 @@ async fn dispatch_fallback( let method = req.method().clone(); if method == Method::GET && path.starts_with("/static/tsjs=") { - return handle_tsjs_dynamic(&req, &state.registry); + return handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SMaxageFallback); } if state.registry.has_route(&method, &path) { @@ -222,6 +225,7 @@ async fn dispatch_fallback( &mut ec_context, auction, req, + EdgeCacheHeader::SMaxageFallback, ) .await?; // Async finalize so the dispatched auction is collected and its bids are @@ -259,6 +263,8 @@ enum NamedRouteHandler { TrustedServerDiscovery, VerifySignature, AdminNotSupported, + AdminEcNotSupported, + AdminEidsLookup, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, @@ -286,7 +292,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_routes() -> [NamedRoute; 13] { +fn named_routes() -> [NamedRoute; 16] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -311,6 +317,26 @@ fn named_routes() -> [NamedRoute; 13] { primary_methods: &[Method::POST], handler: NamedRouteHandler::AdminNotSupported, }, + // Admin EC lookup routes. Registered explicitly (like the key routes + // above) so they never fall through to the publisher fallback, and + // they match `Settings::ADMIN_ENDPOINTS` for auth coverage. + NamedRoute { + path: "/_ts/admin/ec", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcNotSupported, + }, + NamedRoute { + path: "/_ts/admin/ec/{id}", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcNotSupported, + }, + // Admin EIDs echo: pure request inspection (no KV), so the dev + // server serves the real handler. + NamedRoute { + path: "/_ts/admin/eids", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEidsLookup, + }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with // a 404, matching the Fastly and Cloudflare adapters: the production // basic-auth handler regex `^/_ts/admin` does not match them, and letting @@ -407,6 +433,26 @@ fn named_route_handler( ); Ok(resp) } + NamedRouteHandler::AdminEcNotSupported => { + // The EC identity graph is Fastly KV backed; the Axum + // dev server has no store to read. + let body = edgezero_core::body::Body::from( + "Admin EC lookup is not supported on the Axum dev server.\n\ + Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", + ); + let mut resp = Response::new(body); + *resp.status_mut() = StatusCode::NOT_IMPLEMENTED; + resp.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + Ok(resp) + } + NamedRouteHandler::AdminEidsLookup => { + let partner_registry = + PartnerRegistry::from_config(&state.settings.ec.partners)?; + handle_admin_eids_lookup(&partner_registry, &req) + } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::Auction => { // Build the geo-aware EC context so the auction consent diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index 03caa3d11..f75706f02 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -74,6 +74,9 @@ fn all_explicit_routes_are_registered() { ("POST", "/verify-signature"), ("POST", "/_ts/admin/keys/rotate"), ("POST", "/_ts/admin/keys/deactivate"), + ("GET", "/_ts/admin/ec"), + ("GET", "/_ts/admin/ec/{id}"), + ("GET", "/_ts/admin/eids"), ("POST", "/admin/keys/rotate"), ("POST", "/admin/keys/deactivate"), ("POST", "/auction"), @@ -208,6 +211,42 @@ async fn tsjs_route_prefix_is_handled_not_5xx() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tsjs_route_matching_hash_uses_s_maxage_fallback() { + let mut svc = make_service(); + let src = trusted_server_core::tsjs::tsjs_script_src(&["creative"]); + let req = Request::builder() + .method("GET") + .uri(src) + .body(AxumBody::empty()) + .expect("should build request"); + + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + + assert_eq!( + resp.status().as_u16(), + 200, + "matching TSJS hash should serve OK" + ); + assert_eq!( + resp.headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("public, max-age=31536000, s-maxage=31536000, immutable"), + "Axum adapter should render the portable s-maxage fallback" + ); + assert!( + resp.headers().get("surrogate-control").is_none(), + "s-maxage fallback must not emit Fastly Surrogate-Control" + ); +} + // --------------------------------------------------------------------------- // Middleware tests // --------------------------------------------------------------------------- @@ -267,6 +306,63 @@ async fn admin_route_without_credentials_returns_401() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_ec_routes_return_501() { + // The EC identity graph is Fastly KV backed, so the Axum dev server + // answers the admin EC lookup routes locally with 501 instead of letting + // them fall through to the publisher fallback. + let sample_ec_id = format!("{}.abc123", "a".repeat(64)); + for path in [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{sample_ec_id}"), + ] { + let mut svc = make_service(); + let req = Request::builder() + .method("GET") + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(AxumBody::empty()) + .expect("should build request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + assert_eq!( + resp.status().as_u16(), + 501, + "{path} should report that Axum EC lookup is unsupported" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_eids_route_returns_200() { + // The EIDs echo is pure request inspection (no KV), so the dev server + // serves the real handler. + let mut svc = make_service(); + let req = Request::builder() + .method("GET") + .uri("/_ts/admin/eids") + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(AxumBody::empty()) + .expect("should build request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + assert_eq!( + resp.status().as_u16(), + 200, + "/_ts/admin/eids should serve the real EIDs echo handler" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { // Regression for the credential-leak finding: the production basic-auth regex diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 644676fc5..ea593039c 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -10,9 +10,12 @@ use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::cache_policy::EdgeCacheHeader; #[cfg(target_arch = "wasm32")] use trusted_server_core::config_payload::settings_from_config_blob; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; use trusted_server_core::platform::RuntimeServices; @@ -249,6 +252,20 @@ fn admin_key_management_not_supported() -> Response { response } +fn admin_ec_lookup_not_supported() -> Response { + let body = edgezero_core::body::Body::from( + "Admin EC lookup is not supported on Cloudflare Workers.\n\ + Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", + ); + let mut response = Response::new(body); + *response.status_mut() = StatusCode::NOT_IMPLEMENTED; + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + response +} + /// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` /// aliases on the Cloudflare adapter. /// @@ -380,7 +397,11 @@ fn build_router(state: &Arc) -> RouterService { let allow_tsjs = method == Method::GET; let result = if allow_tsjs && path.starts_with("/static/tsjs=") { - handle_tsjs_dynamic(&req, &state.registry) + handle_tsjs_dynamic( + &req, + &state.registry, + EdgeCacheHeader::CloudflareCdnCacheControl, + ) } else if state.registry.has_route(&method, &path) { let mut ec_context = EcContext::default(); state @@ -414,6 +435,7 @@ fn build_router(state: &Arc) -> RouterService { &mut ec_context, auction, req, + EdgeCacheHeader::CloudflareCdnCacheControl, ) .await { @@ -474,6 +496,26 @@ fn build_router(state: &Arc) -> RouterService { .post("/_ts/admin/keys/deactivate", |_ctx: RequestContext| async { Ok::(admin_key_management_not_supported()) }) + // Admin EC lookup routes. Registered explicitly (like the key + // routes above) so they never fall through to the publisher + // fallback, and they match `Settings::ADMIN_ENDPOINTS` for auth + // coverage. The EC identity graph is Fastly KV backed, so this + // adapter has no store to read. + .get("/_ts/admin/ec", |_ctx: RequestContext| async { + Ok::(admin_ec_lookup_not_supported()) + }) + .get("/_ts/admin/ec/{id}", |_ctx: RequestContext| async { + Ok::(admin_ec_lookup_not_supported()) + }) + // Admin EIDs echo: pure request inspection (no KV), so this + // adapter serves the real handler. + .get( + "/_ts/admin/eids", + make_handler(Arc::clone(&state), |s, _services, req| async move { + let partner_registry = PartnerRegistry::from_config(&s.settings.ec.partners)?; + handle_admin_eids_lookup(&partner_registry, &req) + }), + ) .post( "/auction", make_handler(Arc::clone(&state), |s, services, req| async move { diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index 09e3ed324..95c3dcd53 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -203,6 +203,43 @@ async fn tsjs_route_is_routed_not_5xx() { assert!(status < 500, "tsjs route must not 5xx: got {status}"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tsjs_route_emits_cloudflare_cache_header_for_matching_hash() { + let router = test_router(); + let src = trusted_server_core::tsjs::tsjs_script_src(&["creative"]); + let req = request_builder() + .method("GET") + .uri(src) + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + + let resp = route(router, req).await; + + assert_eq!( + resp.status().as_u16(), + 200, + "matching TSJS hash should serve OK" + ); + assert_eq!( + resp.headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("public, max-age=31536000, immutable"), + "browser cache policy should be immutable for matching TSJS hash" + ); + assert_eq!( + resp.headers() + .get("cloudflare-cdn-cache-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=31536000"), + "Cloudflare adapter should emit the Cloudflare-specific edge header" + ); + assert!( + resp.headers().get("surrogate-control").is_none(), + "Cloudflare adapter must not emit Fastly Surrogate-Control" + ); +} + /// Verify that every expected explicit route is registered in the route table. /// /// Uses [`RouterService::routes()`] for introspection rather than checking @@ -215,6 +252,9 @@ fn all_explicit_routes_are_registered() { ("POST", "/verify-signature"), ("POST", "/_ts/admin/keys/rotate"), ("POST", "/_ts/admin/keys/deactivate"), + ("GET", "/_ts/admin/ec"), + ("GET", "/_ts/admin/ec/{id}"), + ("GET", "/_ts/admin/eids"), ("POST", "/auction"), // SPA re-auction endpoint, plus its deprecated `/__ts/` alias. Both // paths are spelled out as literals rather than referencing @@ -275,6 +315,51 @@ async fn authenticated_admin_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_ec_routes_return_501() { + // The EC identity graph is Fastly KV backed, so Cloudflare answers the + // admin EC lookup routes locally with 501 instead of letting them fall + // through to the publisher fallback. + let sample_ec_id = format!("{}.abc123", "a".repeat(64)); + for path in [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{sample_ec_id}"), + ] { + let req = request_builder() + .method("GET") + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 501, + "{path} should report that Cloudflare EC lookup is unsupported" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_eids_route_returns_200() { + // The EIDs echo is pure request inspection (no KV), so this adapter + // serves the real handler. + let req = request_builder() + .method("GET") + .uri("/_ts/admin/eids") + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 200, + "/_ts/admin/eids should serve the real EIDs echo handler" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_route_without_credentials_returns_401() { let router = test_router(); diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index b6bc0f1a1..3d42ae388 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -15,9 +15,11 @@ async-trait = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } +derive_more = { workspace = true } edgezero-adapter-fastly = { workspace = true, features = ["fastly"] } edgezero-core = { workspace = true } error-stack = { workspace = true } +esi = { git = "https://github.com/stackpop/esi.git", rev = "4c53feab4d22ad9a84641b4c46f3f63bc6d197e2" } fastly = { workspace = true } fern = { workspace = true } futures = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index d6090c983..b419e8d54 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -22,6 +22,9 @@ //! | POST | `/verify-signature` | [`handle_verify_signature`] | //! | POST | `/_ts/admin/keys/rotate` | [`handle_rotate_key`] | //! | POST | `/_ts/admin/keys/deactivate` | [`handle_deactivate_key`] | +//! | GET | `/_ts/admin/ec` | [`handle_admin_ec_lookup`] | +//! | GET | `/_ts/admin/ec/{id}` | [`handle_admin_ec_lookup`] | +//! | GET | `/_ts/admin/eids` | [`handle_admin_eids_lookup`] | //! | POST | `/_ts/api/v1/batch-sync` | [`handle_batch_sync`] | //! | GET | `/_ts/api/v1/identify` | [`handle_identify`] | //! | GET | `/_ts/set-tester` | [`handle_set_tester`] | @@ -97,8 +100,10 @@ use error_stack::Report; use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::{handle_admin_ec_lookup, handle_admin_eids_lookup}; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; @@ -257,6 +262,11 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime .config_store(Arc::new(FastlyPlatformConfigStore)) .secret_store(Arc::new(FastlyPlatformSecretStore)) .kv_store(Arc::clone(&state.default_kv_store)) + // Spike-only (#1009). Constructed unconditionally, but only read when the + // assembly mode is a shared-template one — which defaults to Inline, so this + // is inert until an operator opts in. + .template_cache(Arc::new(crate::template_cache::FastlyTemplateCache::new())) + .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) .backend(Arc::new(FastlyPlatformBackend)) .http_client(Arc::new(FastlyPlatformHttpClient)) .geo(Arc::new(FastlyPlatformGeo)) @@ -574,6 +584,18 @@ async fn run_named_route( } NamedRouteHandler::RotateKey => handle_rotate_key(&state.settings, services, req), NamedRouteHandler::DeactivateKey => handle_deactivate_key(&state.settings, services, req), + NamedRouteHandler::AdminEcLookup => { + // Deliberately NOT `ec.kv_graph`: that copy is bot-gated (None for + // non-browser clients), and operators hit this auth-gated endpoint + // with curl. Build the graph directly from settings instead. + let kv = crate::maybe_identity_graph(&state.settings); + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + handle_admin_ec_lookup(kv.as_ref(), &partner_registry, &req) + } + NamedRouteHandler::AdminEidsLookup => { + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + handle_admin_eids_lookup(&partner_registry, &req) + } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::BatchSync => { // Dispatched by execute_named before EC state is built. @@ -735,7 +757,7 @@ async fn dispatch_fallback( }; let result = if uses_dynamic_tsjs_fallback(&method, &path) { - handle_tsjs_dynamic(&req, &state.registry) + handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SurrogateControl) } else if state.registry.has_route(&method, &path) { // Integration-proxy responses are not bounded by // publisher.max_buffered_body_bytes. Publisher fallback below uses the @@ -808,6 +830,7 @@ async fn dispatch_fallback( &mut ec.ec_context, auction, req, + EdgeCacheHeader::SurrogateControl, ) .await { @@ -1001,6 +1024,8 @@ enum NamedRouteHandler { VerifySignature, RotateKey, DeactivateKey, + AdminEcLookup, + AdminEidsLookup, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, @@ -1053,6 +1078,25 @@ const NAMED_ROUTES: &[NamedRoute] = &[ primary_methods: &[Method::POST], handler: NamedRouteHandler::DeactivateKey, }, + // Admin EC lookup: the bare route reads the EC ID from the caller's + // `ts-ec` cookie; the parameterized route takes an explicit EC ID. + NamedRoute { + path: "/_ts/admin/ec", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcLookup, + }, + NamedRoute { + path: "/_ts/admin/ec/{id}", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcLookup, + }, + // Admin EIDs echo: decodes the request's ts-eids/sharedId cookies with + // an ingestion preview. Pure request inspection — no KV access. + NamedRoute { + path: "/_ts/admin/eids", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEidsLookup, + }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with a // 404 instead of executing key operations: the production basic-auth handler // regex `^/_ts/admin` does not match them, and letting them fall through to @@ -1239,12 +1283,15 @@ mod tests { use super::{ AppState, NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, - TrustedServerApp, build_state_from_settings, startup_error_router, + TrustedServerApp, build_per_request_services, build_state_from_settings, + startup_error_router, }; use bytes::Bytes; use edgezero_core::body::Body; + use edgezero_core::context::RequestContext; use edgezero_core::http::{Method, Response, StatusCode, header, request_builder}; use edgezero_core::key_value_store::NoopKvStore; + use edgezero_core::params::PathParams; use edgezero_core::router::RouterService; use std::net::{IpAddr, Ipv4Addr}; use std::sync::Mutex; @@ -1379,6 +1426,36 @@ mod tests { TrustedServerApp::routes_for_state(&state) } + #[test] + fn per_request_services_register_the_fastly_template_assembler() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + let context = RequestContext::new( + empty_request(Method::GET, "/article"), + PathParams::default(), + ); + + let services = build_per_request_services(&state, &context); + let template = format!( + "article{}", + trusted_server_core::publisher::AD_ASSEMBLY_SEAM + ); + let fragment = b""; + let assembled = services + .template_assembler() + .assemble(template.as_bytes(), fragment) + .expect("Fastly services should provide ESI assembly"); + + assert_eq!( + assembled, + template + .replace( + trusted_server_core::publisher::AD_ASSEMBLY_SEAM, + std::str::from_utf8(fragment).expect("fragment should be UTF-8") + ) + .into_bytes() + ); + } + /// Builds a router whose `AppState` uses a registry containing the given /// request filters (and no routes), so dispatch-level request-filter /// behavior can be exercised without a real integration. @@ -1651,6 +1728,44 @@ mod tests { } } + #[test] + fn admin_ec_lookup_routes_are_registered() { + // Both lookup shapes must be explicitly routed to the admin EC + // handler: the bare cookie-based route and the parameterized route. + // Leaving either unrouted would fall through to the publisher + // fallback, forwarding the caller's `Authorization` header to the + // origin. + for path in ["/_ts/admin/ec", "/_ts/admin/ec/{id}"] { + let route = NAMED_ROUTES + .iter() + .find(|route| route.path == path) + .unwrap_or_else(|| panic!("{path} must be a named route")); + assert!( + matches!(route.handler, NamedRouteHandler::AdminEcLookup), + "{path} must map to the admin EC lookup handler" + ); + assert_eq!( + route.primary_methods, + &[Method::GET], + "{path} must have GET as its only primary method" + ); + } + + let eids_route = NAMED_ROUTES + .iter() + .find(|route| route.path == "/_ts/admin/eids") + .expect("should register /_ts/admin/eids as a named route"); + assert!( + matches!(eids_route.handler, NamedRouteHandler::AdminEidsLookup), + "/_ts/admin/eids must map to the admin EIDs lookup handler" + ); + assert_eq!( + eids_route.primary_methods, + &[Method::GET], + "/_ts/admin/eids must have GET as its only primary method" + ); + } + #[test] fn page_bids_serves_canonical_path_and_deprecated_alias() { // The SPA re-auction endpoint lives at the canonical single-underscore diff --git a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs new file mode 100644 index 000000000..4a92c6f78 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs @@ -0,0 +1,287 @@ +//! Fastly cold-response assembly backed by the repaired `stackpop/esi` parser. +//! +//! C2 stores an inert marker. This module creates one synthetic ESI include only in a +//! request-private working copy, resolves it from an already-built fragment, and never +//! performs an HTTP request. + +use std::io::Cursor; + +use esi::{CacheConfig, Configuration, DcaMode, PendingFragmentContent, Processor}; +use fastly::http::StatusCode; +use fastly::{Request, Response}; +use trusted_server_core::platform::{PlatformTemplateAssembler, TemplateAssemblyError}; +use trusted_server_core::publisher::AD_ASSEMBLY_SEAM; + +const INTERNAL_FRAGMENT_PATH: &str = "/_ts/internal/reader-ad-state"; +const SYNTHETIC_ESI_INCLUDE: &[u8] = b""; + +/// Why the Fastly ESI adapter refused or failed to assemble a document. +#[derive(Debug, derive_more::Display)] +enum EsiAssemblyError { + /// The inert seam marker was missing or repeated. + #[display("expected exactly one inert seam marker, found {count}")] + InvalidMarkerCount { count: usize }, + /// Publisher bytes contained ESI instructions outside TS's synthetic seam. + #[display("publisher-authored ESI directives are not allowed")] + PublisherEsiDirective, + /// The parser dispatched a URL other than TS's one synthetic fragment. + #[display("unexpected fragment request path `{path}` (query present: {has_query})")] + UnexpectedFragmentRequest { path: String, has_query: bool }, + /// The pinned parser could not process the document. + #[display("ESI processing failed: {message}")] + Processing { message: String }, + /// The parser changed bytes outside the one synthetic include. + #[display("ESI output was not an exact seam substitution")] + OutputMismatch, +} + +impl core::error::Error for EsiAssemblyError {} + +/// ESI configuration with every cache- and recursion-sensitive option explicit. +fn assembly_configuration() -> Configuration { + Configuration::default() + .with_escaped(false) + .with_default_dca(DcaMode::None) + .with_inherit_parent_dca(false) + .with_max_include_depth(1) + .with_edge_control(false) + .with_caching(CacheConfig { + is_includes_cacheable: false, + includes_default_ttl: None, + includes_force_ttl: None, + is_rendered_cacheable: false, + rendered_cache_control: false, + rendered_ttl: None, + }) +} + +fn contains_esi_directive(bytes: &[u8]) -> bool { + bytes + .windows(b" Result<(Vec, usize), EsiAssemblyError> { + let marker = AD_ASSEMBLY_SEAM.as_bytes(); + let positions = template + .windows(marker.len()) + .enumerate() + .filter_map(|(at, window)| (window == marker).then_some(at)) + .collect::>(); + if positions.len() != 1 { + return Err(EsiAssemblyError::InvalidMarkerCount { + count: positions.len(), + }); + } + if contains_esi_directive(template) { + return Err(EsiAssemblyError::PublisherEsiDirective); + } + + let at = positions[0]; + let mut working = + Vec::with_capacity(template.len() - marker.len() + SYNTHETIC_ESI_INCLUDE.len()); + working.extend_from_slice(&template[..at]); + working.extend_from_slice(SYNTHETIC_ESI_INCLUDE); + working.extend_from_slice(&template[at + marker.len()..]); + Ok((working, at)) +} + +fn completed_fragment_response( + request: &Request, + fragment: &[u8], +) -> Result { + let path = request.get_path().to_string(); + let has_query = request.get_url().query().is_some(); + if path != INTERNAL_FRAGMENT_PATH || has_query { + return Err(EsiAssemblyError::UnexpectedFragmentRequest { path, has_query }); + } + + Ok(PendingFragmentContent::CompletedRequest(Box::new( + Response::from_status(StatusCode::OK) + .with_header( + fastly::http::header::CONTENT_TYPE, + "text/html; charset=utf-8", + ) + .with_body(fragment.to_vec()), + ))) +} + +fn assemble_with_observer( + template: &[u8], + fragment: &[u8], + on_dispatch: F, +) -> Result, EsiAssemblyError> +where + F: Fn() + 'static, +{ + let (working, seam_at) = template_with_synthetic_include(template)?; + let fragment_len = fragment.len(); + let fragment_response = fragment.to_vec(); + let dispatcher = move |request, _index| { + on_dispatch(); + completed_fragment_response(&request, &fragment_response) + .map_err(|error| esi::ESIError::FragmentRequestError(error.to_string())) + }; + let mut processor = Processor::new(None, assembly_configuration()); + let mut output = Vec::with_capacity(template.len() + fragment_len); + processor + .process_stream(Cursor::new(working), &mut output, Some(&dispatcher), None) + .map_err(|error| EsiAssemblyError::Processing { + message: error.to_string(), + })?; + let expected_len = template.len() - AD_ASSEMBLY_SEAM.len() + fragment_len; + let output_tail_at = seam_at + fragment_len; + let template_tail_at = seam_at + AD_ASSEMBLY_SEAM.len(); + if output.len() != expected_len + || output[..seam_at] != template[..seam_at] + || &output[seam_at..output_tail_at] != fragment + || output[output_tail_at..] != template[template_tail_at..] + { + return Err(EsiAssemblyError::OutputMismatch); + } + Ok(output) +} + +fn assemble(template: &[u8], fragment: &[u8]) -> Result, EsiAssemblyError> { + assemble_with_observer(template, fragment, || {}) +} + +/// Fastly implementation of the core cold-response assembly boundary. +pub struct FastlyTemplateAssembler; + +impl PlatformTemplateAssembler for FastlyTemplateAssembler { + fn assemble(&self, template: &[u8], fragment: &[u8]) -> Result, TemplateAssemblyError> { + assemble(template, fragment).map_err(|error| TemplateAssemblyError::Failed { + message: error.to_string(), + }) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + use trusted_server_core::publisher::AD_ASSEMBLY_SEAM; + + const FRAGMENT: &[u8] = b""; + + fn template(body: &str) -> Vec { + format!("{body}{AD_ASSEMBLY_SEAM}").into_bytes() + } + + #[test] + fn a_script_larger_than_the_parser_chunk_survives_exactly() { + let script = format!( + "", + "x".repeat(120_000) + ); + let document = template(&script); + let dispatches = Arc::new(AtomicUsize::new(0)); + let observed_dispatches = Arc::clone(&dispatches); + + let assembled = assemble_with_observer(&document, FRAGMENT, move || { + observed_dispatches.fetch_add(1, Ordering::Relaxed); + }) + .expect("should assemble a document with a large script"); + + let seam_at = document + .windows(AD_ASSEMBLY_SEAM.len()) + .position(|window| window == AD_ASSEMBLY_SEAM.as_bytes()) + .expect("should find seam"); + let mut expected = Vec::new(); + expected.extend_from_slice(&document[..seam_at]); + expected.extend_from_slice(FRAGMENT); + expected.extend_from_slice(&document[seam_at + AD_ASSEMBLY_SEAM.len()..]); + + assert_eq!( + assembled, expected, + "ESI must alter only the synthetic seam" + ); + assert_eq!(dispatches.load(Ordering::Relaxed), 1); + } + + #[test] + fn missing_and_repeated_markers_are_rejected_before_parsing() { + let missing = assemble(b"plain", FRAGMENT) + .expect_err("should reject a missing marker"); + let repeated = assemble( + format!("{AD_ASSEMBLY_SEAM}{AD_ASSEMBLY_SEAM}").as_bytes(), + FRAGMENT, + ) + .expect_err("should reject repeated markers"); + + assert!(matches!( + missing, + EsiAssemblyError::InvalidMarkerCount { count: 0 } + )); + assert!(matches!( + repeated, + EsiAssemblyError::InvalidMarkerCount { count: 2 } + )); + } + + #[test] + fn every_publisher_esi_directive_form_is_rejected_case_insensitively() { + for directive in [ + "", + "secret", + "x", + "$(HTTP_HOST)", + "text", + "", + ] { + let error = assemble(&template(directive), FRAGMENT) + .expect_err("should reject publisher-authored ESI"); + + assert!(matches!(error, EsiAssemblyError::PublisherEsiDirective)); + } + } + + #[test] + fn fragment_esi_is_emitted_verbatim_and_never_reparsed() { + let fragment = b""; + + let assembled = assemble(&template("article"), fragment).expect("should assemble"); + + assert!( + assembled + .windows(fragment.len()) + .any(|window| window == fragment), + "fragment bytes must remain data" + ); + } + + #[test] + fn dispatcher_rejects_every_url_except_the_synthetic_internal_one() { + let unexpected = fastly::Request::get("https://example.com/not-the-seam"); + let with_query = + fastly::Request::get("https://example.com/_ts/internal/reader-ad-state?publisher=1"); + + assert!(matches!( + completed_fragment_response(&unexpected, FRAGMENT), + Err(EsiAssemblyError::UnexpectedFragmentRequest { .. }) + )); + assert!(matches!( + completed_fragment_response(&with_query, FRAGMENT), + Err(EsiAssemblyError::UnexpectedFragmentRequest { .. }) + )); + } + + #[test] + fn configuration_cannot_cache_or_reparse_reader_state() { + let configuration = assembly_configuration(); + + assert!(!configuration.cache.is_includes_cacheable); + assert!(configuration.cache.includes_default_ttl.is_none()); + assert!(configuration.cache.includes_force_ttl.is_none()); + assert!(!configuration.cache.is_rendered_cacheable); + assert!(!configuration.cache.rendered_cache_control); + assert!(configuration.cache.rendered_ttl.is_none()); + assert_eq!(configuration.default_dca, DcaMode::None); + assert!(!configuration.inherit_parent_dca); + assert_eq!(configuration.max_include_depth, 1); + assert!(!configuration.enable_edge_control); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 39d35b198..85de2195b 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -11,6 +11,7 @@ use error_stack::Report; use fastly::http::Method as FastlyMethod; use fastly::{Request as FastlyRequest, Response as FastlyResponse}; +use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; @@ -29,11 +30,13 @@ mod app; mod backend; mod compat; mod ec_kv; +mod esi_assembly; mod logging; mod management_api; mod middleware; mod platform; mod rate_limiter; +mod template_cache; mod tinybird; use crate::app::{EcFinalizeState, TrustedServerApp, load_settings_from_config_store}; @@ -202,7 +205,7 @@ fn edgezero_main(mut req: FastlyRequest) { } if let Some(policy) = asset_cache_policy { - policy.apply_after_route_finalization(&mut response); + policy.apply_after_route_finalization(&mut response, EdgeCacheHeader::SurrogateControl); } if let Some(ec_state) = ec_state { @@ -328,14 +331,8 @@ fn send_edgezero_response( mut response: HttpResponse, request_filter_effects: Option<&RequestFilterEffects>, ) { - if let Some(effects) = request_filter_effects { - effects.apply_to_response(&mut response); - } - - // Final cache guard: EC finalization and request-filter effects may have - // added a per-user Set-Cookie after `apply_finalize_headers` ran, so - // re-apply the privacy downgrade before send. - crate::middleware::enforce_set_cookie_cache_privacy(&mut response); + apply_terminal_response_effects(&mut response, request_filter_effects); + crate::middleware::enforce_uncacheable_cache_privacy(&mut response); let (parts, body) = response.into_parts(); @@ -364,6 +361,26 @@ fn send_edgezero_response( } } +/// Apply every late response mutation, then restore privacy invariants before headers commit. +fn apply_terminal_response_effects( + response: &mut HttpResponse, + request_filter_effects: Option<&RequestFilterEffects>, +) { + let must_remain_private = + trusted_server_core::response_privacy::is_private_or_no_store(response.headers()); + if let Some(effects) = request_filter_effects { + effects.apply_to_response(response); + } + if must_remain_private { + trusted_server_core::response_privacy::enforce_private_no_store(response); + } + + // Final cache guard: EC finalization and request-filter effects may have + // added a per-user Set-Cookie after `apply_finalize_headers` ran, so + // re-apply the privacy downgrade before send. + crate::middleware::enforce_set_cookie_cache_privacy(response); +} + const FALLBACK_UNAVAILABLE: &str = "unavailable"; const FALLBACK_NOT_SENT: &str = "not sent"; const FALLBACK_NONE: &str = "none"; @@ -485,6 +502,7 @@ mod tests { use edgezero_core::http::HeaderValue; use edgezero_core::http::response_builder; use fastly::mime; + use trusted_server_core::integrations::HeaderMutation; fn test_settings() -> Settings { Settings::from_toml( @@ -557,6 +575,36 @@ mod tests { ); } + #[test] + fn late_filter_effects_cannot_make_an_assembled_response_public() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .header("etag", "\"reader-document\"") + .body(EdgeBody::empty()) + .expect("should build response"); + let effects = RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![ + HeaderMutation::set("cache-control", "public, s-maxage=3600"), + HeaderMutation::set("surrogate-control", "max-age=3600"), + HeaderMutation::set("cdn-cache-control", "public, max-age=3600"), + ], + }; + + apply_terminal_response_effects(&mut response, Some(&effects)); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("private, no-store") + ); + assert!(response.headers().get("surrogate-control").is_none()); + assert!(response.headers().get("cdn-cache-control").is_none()); + assert!(response.headers().get("etag").is_none()); + } + #[test] #[allow(clippy::panic)] fn entry_point_finalize_skips_geo_lookup_for_401() { diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 2c00ac2ff..153d90295 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -235,7 +235,9 @@ pub(crate) fn apply_finalize_headers( /// entry point (`main.rs`) can re-apply it after /// [`ec_finalize_response`](trusted_server_core::ec::finalize::ec_finalize_response) /// writes the EC identity `Set-Cookie`, using the single shared implementation. -pub(crate) use trusted_server_core::response_privacy::enforce_set_cookie_cache_privacy; +pub(crate) use trusted_server_core::response_privacy::{ + enforce_set_cookie_cache_privacy, enforce_uncacheable_cache_privacy, +}; // --------------------------------------------------------------------------- // Tests @@ -496,6 +498,29 @@ mod tests { ); } + #[test] + fn enforce_uncacheable_cache_privacy_handles_late_filter_headers() { + let mut response = response_with_headers(&[ + ("cache-control", "private, max-age=0"), + ("surrogate-control", "max-age=600"), + ]); + + enforce_uncacheable_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "should preserve the late private directive" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip the normalized edge header after late filter effects" + ); + } + // --------------------------------------------------------------------------- // FinalizeResponseMiddleware::handle tests // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs new file mode 100644 index 000000000..3f168f574 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -0,0 +1,505 @@ +//! Fastly Core Cache backing for the shared transformed-template cache (C2). +//! +//! Only the Fastly adapter implements this; every other adapter uses +//! `UnavailableTemplateCache`, so the ESI assembly mode stays portable and only +//! the caching is Fastly-only. +//! +//! **Why Core Cache and not read-through caching.** Read-through with `after_send` + +//! `set_body_transform` looks like a better fit — it keeps HTTP semantics and derives +//! TTL and surrogate keys from origin headers for free. It is unreachable here: +//! Viceroy 0.17 stubs the entire HTTP Cache ABI and the SDK converts that into a +//! *send error*, so setting `after_send` makes every publisher origin fetch fail +//! under `fastly compute serve`, `cargo test-fastly` and the parity suite. It is also +//! silently dead whenever the origin request is in pass mode, and its closure bounds +//! (`Fn + Send + Sync`) are incompatible with a platform layer that is `!Send` by +//! construction. Recorded in the spike plan's Task 3 Step 4 so nobody re-proposes it. +//! +//! Spike-only. Remove with the spike. + +use fastly::cache::core::{CacheKey, Found, Transaction}; +use std::io::Write as _; +use std::time::Duration; +use trusted_server_core::platform::{ + PlatformTemplateCache, PlatformTemplateCacheReservation, TemplateCacheError, TemplateCacheKey, + TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, + TemplateMetadata, +}; + +/// Surrogate key attached to every stored template, so a single purge clears them +/// all. This is the rollback lever: without it, backing out a bad template means +/// waiting for the TTL. +const PURGE_ALL_SURROGATE_KEY: &str = "ts-template"; + +/// Fastly Core Cache implementation of the C2 template cache. +#[derive(Default)] +pub struct FastlyTemplateCache; + +impl FastlyTemplateCache { + /// Create the Fastly Core Cache implementation. + /// + /// Entry lifetime is supplied per insert after core validates origin freshness + /// and applies the operator's configured safety ceiling. + #[must_use] + pub const fn new() -> Self { + Self + } +} + +fn backend_error(message: impl Into) -> TemplateCacheError { + TemplateCacheError::Backend { + message: message.into(), + } +} + +enum ReadFoundError { + Invalid(TemplateCacheMiss), + Backend(TemplateCacheError), +} + +fn read_found(found: &Found, key: &TemplateCacheKey) -> Result { + if found.is_stale() { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::NotFound)); + } + + let metadata = TemplateMetadata::decode(&found.user_metadata()).ok_or( + ReadFoundError::Invalid(TemplateCacheMiss::UnreadableMetadata), + )?; + if metadata.schema_version != key.schema_version { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::SchemaMismatch)); + } + if found + .known_length() + .is_some_and(|length| length != metadata.body_len) + { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::Truncated)); + } + + let body = found + .to_stream() + .map_err(|error| { + ReadFoundError::Backend(backend_error(format!( + "opening cached template body failed: {error:?}" + ))) + })? + .into_bytes(); + if body.len() as u64 != metadata.body_len { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::Truncated)); + } + Ok(TemplateEntry { metadata, body }) +} + +struct FastlyTemplateReservation { + transaction: Transaction, + surrogate_keys: Vec, +} + +impl PlatformTemplateCacheReservation for FastlyTemplateReservation { + fn insert( + self: Box, + metadata: &TemplateMetadata, + body: Vec, + max_age: Duration, + ) -> Result<(), TemplateCacheError> { + if metadata.body_len != body.len() as u64 { + return Err(backend_error(format!( + "metadata body_len {} does not match the {} bytes supplied", + metadata.body_len, + body.len() + ))); + } + + let mut writer = self + .transaction + .insert(max_age) + .surrogate_keys(self.surrogate_keys.iter().map(String::as_str)) + .known_length(body.len() as u64) + .user_metadata(metadata.encode().into()) + .execute() + .map_err(|e| backend_error(format!("cache insert failed: {e:?}")))?; + writer + .write_all(&body) + .map_err(|e| backend_error(format!("writing template body failed: {e}")))?; + writer + .finish() + .map_err(|e| backend_error(format!("finishing the cached template failed: {e}")))?; + Ok(()) + } + + fn cancel(self: Box) -> Result<(), TemplateCacheError> { + self.transaction + .cancel_insert_or_update() + .map_err(|e| backend_error(format!("cancelling cache reservation failed: {e:?}"))) + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformTemplateCache for FastlyTemplateCache { + async fn lookup_or_reserve( + &self, + key: &TemplateCacheKey, + ) -> Result { + let transaction = Transaction::lookup(CacheKey::from(key.to_cache_key().into_bytes())) + .execute() + .map_err(|e| backend_error(format!("transactional lookup failed: {e:?}")))?; + + if transaction.must_insert_or_update() { + return Ok(TemplateCacheLookup::Reserved( + TemplateCacheReservation::new(Box::new(FastlyTemplateReservation { + transaction, + surrogate_keys: key.surrogate_keys(), + })), + )); + } + + let found = transaction.found().ok_or_else(|| { + backend_error("transaction returned neither a hit nor an insert obligation") + })?; + Ok(match read_found(&found, key) { + Ok(entry) => TemplateCacheLookup::Hit(entry), + Err(ReadFoundError::Invalid(miss)) => TemplateCacheLookup::Invalid(miss), + Err(ReadFoundError::Backend(error)) => return Err(error), + }) + } + + async fn get(&self, key: &TemplateCacheKey) -> Result { + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + // A plain lookup, not a transaction: a read that does not intend to insert + // must not take an insert obligation it will never discharge, which would + // block every other client waiting on the same key until they time out. + let found = fastly::cache::core::lookup(cache_key) + .execute() + .map_err(|_| TemplateCacheMiss::NotFound)? + .ok_or(TemplateCacheMiss::NotFound)?; + + read_found(&found, key).map_err(|error| match error { + ReadFoundError::Invalid(miss) => miss, + ReadFoundError::Backend(error) => { + // This legacy method cannot expose a backend error. Production uses + // `lookup_or_reserve`, which preserves it for bounded diagnostics. + log::warn!("c2_template_cache legacy read failed: {error}"); + TemplateCacheMiss::NotFound + } + }) + } + + async fn put( + &self, + key: &TemplateCacheKey, + metadata: &TemplateMetadata, + body: Vec, + max_age: Duration, + ) -> Result<(), TemplateCacheError> { + if metadata.body_len != body.len() as u64 { + return Err(backend_error(format!( + "metadata body_len {} does not match the {} bytes supplied; storing \ + this would make every read a truncation miss", + metadata.body_len, + body.len() + ))); + } + + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + // Transactional insert so a cold key under load transforms once rather than + // once per concurrent request. + let tx = Transaction::lookup(cache_key) + .execute() + .map_err(|e| backend_error(format!("transactional lookup failed: {e:?}")))?; + + // Order matters. A STALE entry sets *both* `found()` and + // `must_insert_or_update()`. Testing `found()` first would return early on + // the stale bytes and never discharge the obligation, leaving every + // concurrent waiter blocked until timeout. + if !tx.must_insert_or_update() { + // Someone else already inserted a fresh entry. Nothing to do, and + // nothing to discharge. + return Ok(()); + } + + // `Transaction::insert` takes `self`, so from here there is no handle left to + // cancel the insert with. A write that fails part-way therefore cannot be + // retracted — which is why `TemplateMetadata::body_len` exists and `get` + // checks it. The metadata is written before the body, so a truncated entry + // still carries the length it was supposed to have. + let surrogate_keys = key.surrogate_keys(); + let mut writer = tx + .insert(max_age) + .surrogate_keys(surrogate_keys.iter().map(String::as_str)) + .user_metadata(metadata.encode().into()) + .execute() + .map_err(|e| backend_error(format!("cache insert failed: {e:?}")))?; + + if let Err(e) = writer.write_all(&body) { + // Deliberately not calling `finish()`. An unfinished entry has no known + // length, and even if it is observable, `get`'s length check rejects it. + return Err(backend_error(format!("writing template body failed: {e}"))); + } + + // Required. Without it the object never completes and its length stays + // unknown, so readers see a partial or absent entry. + writer + .finish() + .map_err(|e| backend_error(format!("finishing the cached template failed: {e}")))?; + + Ok(()) + } + + async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError> { + fastly::http::purge::purge_surrogate_key(&key.url_surrogate_key()) + .map_err(|e| backend_error(format!("purging invalid template failed: {e:?}"))) + } + + async fn purge_all(&self) -> Result<(), TemplateCacheError> { + fastly::http::purge::purge_surrogate_key(PURGE_ALL_SURROGATE_KEY) + .map_err(|e| backend_error(format!("purging templates failed: {e:?}"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use trusted_server_core::creative_opportunities::AssemblyMode; + use trusted_server_core::platform::TEMPLATE_SCHEMA_VERSION; + + /// Distinct per test, so tests sharing the process cache cannot collide. + fn key(url: &str) -> TemplateCacheKey { + TemplateCacheKey { + url: url.to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + origin_identity: "https://origin.example.com\0origin.example.com".to_string(), + assembly_mode: AssemblyMode::Esi, + vary_values: vec![trusted_server_core::platform::VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }], + template_fingerprint: "fp".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + } + } + + fn metadata_for(body: &[u8]) -> TemplateMetadata { + TemplateMetadata { + policy_headers: Vec::new(), + content_encoding: "identity".to_string(), + content_type: "text/html; charset=utf-8".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: body.len() as u64, + } + } + + /// The trait is `async_trait(?Send)` and this crate has no async test runtime, + /// so drive the futures directly. + fn run(fut: impl core::future::Future) -> T { + futures::executor::block_on(fut) + } + + fn cache() -> FastlyTemplateCache { + FastlyTemplateCache::new() + } + + #[test] + fn a_stored_template_reads_back_intact() { + let cache = cache(); + let key = key("https://example.com/roundtrip"); + let body = b"template".to_vec(); + let metadata = metadata_for(&body); + + run(cache.put(&key, &metadata, body.clone(), Duration::from_secs(60))) + .expect("should store"); + + let entry = run(cache.get(&key)).expect("should read back"); + assert_eq!(entry.body, body, "bytes must survive the round trip"); + assert_eq!(entry.metadata, metadata, "metadata must survive too"); + } + + #[test] + fn transactional_lookup_reserves_before_insert_then_hits() { + let cache = cache(); + let key = key("https://example.com/pre-origin-reservation"); + let body = b"collapsed".to_vec(); + let metadata = metadata_for(&body); + + let reservation = match run(cache.lookup_or_reserve(&key)).expect("lookup should work") { + TemplateCacheLookup::Reserved(reservation) => reservation, + _ => panic!("a cold transactional lookup must assign the insert obligation"), + }; + reservation + .insert(&metadata, body.clone(), Duration::from_secs(17)) + .expect("reservation should insert"); + + match run(cache.lookup_or_reserve(&key)).expect("warm lookup should work") { + TemplateCacheLookup::Hit(entry) => assert_eq!(entry.body, body), + _ => panic!("the next transactional lookup must see the inserted template"), + } + } + + #[test] + fn an_absent_key_is_a_miss_not_an_error() { + let miss = + run(cache().get(&key("https://example.com/never-stored"))).expect_err("should miss"); + assert_eq!(miss, TemplateCacheMiss::NotFound); + } + + #[test] + fn a_different_assembly_mode_does_not_read_the_same_entry() { + // The arms emit different bytes. If they shared an entry, one would serve + // the other's template. + let cache = cache(); + let esi = key("https://example.com/mode-split"); + let mut inline = esi.clone(); + inline.assembly_mode = AssemblyMode::Inline; + + let body = b"esi-template".to_vec(); + run(cache.put(&esi, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + + assert_eq!( + run(cache.get(&inline)).err(), + Some(TemplateCacheMiss::NotFound), + "inline must not read the ESI arm's template" + ); + } + + #[test] + fn a_schema_bump_reads_a_miss_rather_than_a_stale_shape() { + let cache = cache(); + let key_v1 = key("https://example.com/schema"); + let body = b"old-shape".to_vec(); + run(cache.put(&key_v1, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + + // A deploy that changes the transform bumps the constant. The old entry must + // not be assembled against. + let mut key_v2 = key_v1.clone(); + key_v2.schema_version = TEMPLATE_SCHEMA_VERSION + 1; + + assert_eq!( + run(cache.get(&key_v2)).err(), + Some(TemplateCacheMiss::NotFound), + "a bumped schema changes the key, so the old entry is simply not found" + ); + } + + #[test] + fn a_stale_but_present_entry_reads_as_a_miss_rather_than_being_served() { + // Stale-while-revalidate is a real option and deliberately not taken: it is a + // state machine `cache::core` does not implement for you, and serving stale here + // means serving a template built by an older transform or an older JS bundle. + // + // The entry has to be *present and stale*, not merely expired. A zero TTL with no + // `stale_while_revalidate` window is simply absent, so a test written that way + // passes without ever reaching `is_stale()` — verified: reverting the staleness + // check left that version green. The revalidate window is what keeps the object + // readable while stale, so this actually exercises the branch. + let key = key("https://example.com/stale"); + let body = b"stale-template".to_vec(); + let metadata = metadata_for(&body); + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + let mut writer = fastly::cache::core::insert(cache_key, Duration::from_secs(0)) + .stale_while_revalidate(Duration::from_secs(60)) + .user_metadata(metadata.encode().into()) + .execute() + .expect("should begin insert"); + writer.write_all(&body).expect("should write body"); + writer.finish().expect("should finish insert"); + + let miss = run(cache().get(&key)).expect_err("a stale template must not be served"); + assert_eq!(miss, TemplateCacheMiss::NotFound); + } + + #[test] + fn purge_all_clears_stored_templates() { + // The rollback lever. Without this, backing out a bad template means waiting + // for the TTL. + let cache = cache(); + let key = key("https://example.com/purge"); + let body = b"template".to_vec(); + run(cache.put(&key, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + run(cache.get(&key)).expect("should be present before purge"); + + run(cache.purge_all()).expect("should purge"); + + assert!( + run(cache.get(&key)).is_err(), + "purge must clear the template, or rollback is TTL-bound" + ); + } + + #[test] + fn a_second_put_on_a_fresh_entry_is_a_no_op() { + // Exercises the `must_insert_or_update` early return: a concurrent writer + // that finds a fresh entry must neither error nor overwrite. + let cache = cache(); + let key = key("https://example.com/second-put"); + let first = b"first".to_vec(); + run(cache.put( + &key, + &metadata_for(&first), + first.clone(), + Duration::from_secs(60), + )) + .expect("first put stores"); + + let second = b"second".to_vec(); + run(cache.put( + &key, + &metadata_for(&second), + second, + Duration::from_secs(60), + )) + .expect("second put should be a no-op, not an error"); + + assert_eq!( + run(cache.get(&key)).expect("should read").body, + first, + "a fresh entry must not be overwritten by a racing writer" + ); + } + + #[test] + fn the_cache_round_trips_through_the_platform_trait_object() { + // Every other test here calls `FastlyTemplateCache` concretely. The publisher + // never does — it reaches the cache as a `dyn PlatformTemplateCache` behind + // `RuntimeServices`. That join is what `app.rs` wires, and until this test it + // was only type-checked, never executed. + let cache: std::sync::Arc = std::sync::Arc::new(cache()); + let key = key("https://example.com/via-trait-object"); + let body = b"template".to_vec(); + + run(cache.put( + &key, + &metadata_for(&body), + body.clone(), + Duration::from_secs(60), + )) + .expect("should store"); + + assert_eq!( + run(cache.get(&key)).expect("should read back").body, + body, + "the trait object must reach the same Core Cache the concrete type does" + ); + } + + #[test] + fn a_length_mismatch_is_refused_at_write_rather_than_stored() { + // Storing metadata whose length disagrees with the body would make every + // subsequent read a truncation miss — a cache that silently never hits. + // Catch it at the write instead. + let cache = cache(); + let key = key("https://example.com/length-mismatch"); + let mut metadata = metadata_for(b"12345"); + metadata.body_len = 999; + + let err = run(cache.put(&key, &metadata, b"12345".to_vec(), Duration::from_secs(60))) + .expect_err("a length mismatch must be refused"); + assert!( + matches!(err, TemplateCacheError::Backend { .. }), + "expected a backend error, got {err:?}" + ); + } +} diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 960bafc41..45a50229a 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -10,7 +10,10 @@ use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; @@ -142,12 +145,15 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 13] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 16] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), ("/_ts/admin/keys/rotate", &[Method::POST]), ("/_ts/admin/keys/deactivate", &[Method::POST]), + ("/_ts/admin/ec", &[Method::GET]), + ("/_ts/admin/ec/{id}", &[Method::GET]), + ("/_ts/admin/eids", &[Method::GET]), ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), ("/auction", &[Method::POST]), @@ -361,6 +367,20 @@ fn admin_key_management_not_supported() -> Response { response } +fn admin_ec_lookup_not_supported() -> Response { + let body = edgezero_core::body::Body::from( + "Admin EC lookup is not supported on Fermyon Spin.\n\ + Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", + ); + let mut response = Response::new(body); + *response.status_mut() = StatusCode::NOT_IMPLEMENTED; + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + response +} + // --------------------------------------------------------------------------- // Error helper // --------------------------------------------------------------------------- @@ -513,6 +533,23 @@ fn build_router(state: &Arc) -> RouterService { Ok::(admin_key_management_not_supported()) }; + let admin_ec_not_supported_handler = |_ctx: RequestContext| async { + Ok::(admin_ec_lookup_not_supported()) + }; + + // Admin EIDs echo: pure request inspection (no KV), so this adapter + // serves the real handler. + let s = Arc::clone(&state); + let admin_eids_handler = move |ctx: RequestContext| { + let s = Arc::clone(&s); + async move { + let req = ctx.into_request(); + let result = PartnerRegistry::from_config(&s.settings.ec.partners) + .and_then(|registry| handle_admin_eids_lookup(®istry, &req)); + Ok::(result.unwrap_or_else(|e| http_error(&e))) + } + }; + // /auction let s = Arc::clone(&state); let auction_handler = move |ctx: RequestContext| { @@ -665,7 +702,7 @@ fn build_router(state: &Arc) -> RouterService { // Dynamic tsjs serving is GET-only; other methods fall through to the // integration/publisher fallback. let result = if method == Method::GET && path.starts_with("/static/tsjs=") { - handle_tsjs_dynamic(&req, &state.registry) + handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SMaxageFallback) } else if state.registry.has_route(&method, &path) { let mut ec_context = EcContext::default(); state @@ -699,6 +736,7 @@ fn build_router(state: &Arc) -> RouterService { &mut ec_context, auction, req, + EdgeCacheHeader::SMaxageFallback, ) .await { @@ -758,6 +796,14 @@ fn build_router(state: &Arc) -> RouterService { // credentials and key-management payloads to the origin. .post("/_ts/admin/keys/rotate", admin_not_supported_handler) .post("/_ts/admin/keys/deactivate", admin_not_supported_handler) + // Admin EC lookup routes. Registered explicitly (like the key + // routes above) so they never fall through to the publisher + // fallback, and they match `Settings::ADMIN_ENDPOINTS` for auth + // coverage. The EC identity graph is Fastly KV backed, so this + // adapter has no store to read. + .get("/_ts/admin/ec", admin_ec_not_supported_handler) + .get("/_ts/admin/ec/{id}", admin_ec_not_supported_handler) + .get("/_ts/admin/eids", admin_eids_handler) .post("/auction", auction_handler) .get(PAGE_BIDS_PATH, page_bids_handler.clone()) .route(PAGE_BIDS_PATH, Method::OPTIONS, page_bids_options_handler) diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 2f7b1037e..07e4d62d3 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -113,6 +113,51 @@ async fn authenticated_admin_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_ec_routes_return_501() { + // The EC identity graph is Fastly KV backed, so Spin answers the admin + // EC lookup routes locally with 501 instead of letting them fall through + // to the publisher fallback. + let sample_ec_id = format!("{}.abc123", "a".repeat(64)); + for path in [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{sample_ec_id}"), + ] { + let req = request_builder() + .method("GET") + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 501, + "{path} should report that Spin EC lookup is unsupported" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_eids_route_returns_200() { + // The EIDs echo is pure request inspection (no KV), so this adapter + // serves the real handler. + let req = request_builder() + .method("GET") + .uri("/_ts/admin/eids") + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 200, + "/_ts/admin/eids should serve the real EIDs echo handler" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn health_route_returns_ok() { // Parity with the Fastly/Axum adapters: GET /health is a cheap liveness probe @@ -209,6 +254,36 @@ async fn tsjs_route_is_routed_not_5xx() { assert!(status < 500, "tsjs route must not 5xx: got {status}"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tsjs_route_matching_hash_uses_s_maxage_fallback() { + let router = test_router(); + let src = trusted_server_core::tsjs::tsjs_script_src(&["creative"]); + let req = request_builder() + .method("GET") + .uri(src) + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + + let resp = route(router, req).await; + + assert_eq!( + resp.status().as_u16(), + 200, + "matching TSJS hash should serve OK" + ); + assert_eq!( + resp.headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("public, max-age=31536000, s-maxage=31536000, immutable"), + "Spin adapter should render the portable s-maxage fallback" + ); + assert!( + resp.headers().get("surrogate-control").is_none(), + "s-maxage fallback must not emit Fastly Surrogate-Control" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn verify_signature_is_routed() { let router = test_router(); diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index fe9c3664b..eff945b3a 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -17,6 +17,7 @@ workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] chromiumoxide = { workspace = true } clap = { workspace = true } +edgezero-core = { workspace = true } edgezero-cli = { workspace = true } futures = { workspace = true } log = { workspace = true } @@ -62,4 +63,5 @@ tokio = { workspace = true, features = ["test-util"] } x509-parser = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +temp-env = { workspace = true } tempfile = { workspace = true } diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs new file mode 100644 index 000000000..215a6a300 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -0,0 +1,807 @@ +//! Pure comparison of configured expected slots against browser ad evidence. +//! +//! This module is collector-independent and Chrome-free: it takes decoded +//! [`BrowserAdEvidence`] plus the [`ExpectedSlot`] set and produces a +//! [`PageVerificationResult`] with per-slot statuses, warnings, and unmatched +//! extra evidence, mirroring spec §5.3–§5.6. +//! +//! Consumed by the browser collector decode (Task 8) and the audit verifier +//! (Task 9); exercised by tests until then, hence the module-scoped allow. +#![allow( + dead_code, + reason = "consumed by the browser collector and audit verifier in later tasks" +)] + +use serde::Deserialize; + +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +use crate::ad_templates::expected::ExpectedSlot; +use crate::ad_templates::output::Warning; + +/// The phase in which a piece of evidence was observed. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhase { + /// Observed during the initial load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A DOM element ID observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct DomEvidence { + /// The element ID. + pub dom_id: String, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// A GPT slot observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct GptSlotEvidence { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `(width, height)` pairs (non-numeric dropped upstream). + pub sizes: Vec<(u32, u32)>, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// An `apstag.fetchBids` call observed on the page (spec §5.5). +#[derive(Debug, Clone, Deserialize)] +pub struct ApsFetchBidsEvidence { + /// The APS slot ID requested. + pub slot_id: String, + /// Sizes requested for the slot. + pub sizes: Vec<(u32, u32)>, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// A `/__ts/page-bids` observation for SPA routes (spec §5.2). +/// +/// DEFERRED in Phase 1: kept as forward scaffolding so the decoded evidence shape +/// stays forward-compatible. Not populated by the collector or surfaced in JSON. +#[derive(Debug, Clone, Deserialize)] +pub struct PageBidsEvidence { + /// The slot ID present in the page-bids response. + pub slot_id: String, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// All read-only ad evidence decoded from a single browser page. +#[derive(Debug, Clone, Deserialize)] +pub struct BrowserAdEvidence { + /// DOM element IDs matching configured prefixes. + pub dom_ids: Vec, + /// GPT slots observed via `defineSlot` and `getSlots()`. + pub gpt_slots: Vec, + /// `apstag.fetchBids` calls observed. + pub aps_calls: Vec, + /// `/__ts/page-bids` observations (deferred; default empty). + #[serde(default)] + pub page_bids: Vec, + /// Collector-level warnings (no page HTML/cookies/storage). + #[serde(default)] + pub warnings: Vec, +} + +/// Summary of the runtime ad-stack gate for a page. +#[derive(Debug, Clone, Copy)] +pub struct RuntimeGateSummary { + /// The three-state ad-stack expectation. + pub expected: RuntimeAdStackExpected, +} + +impl RuntimeGateSummary { + /// Builds a summary from a computed runtime expectation. + #[must_use] + pub fn from_expected(expected: RuntimeAdStackExpected) -> Self { + Self { expected } + } + + #[cfg(test)] + fn unknown_allowed() -> Self { + Self::from_expected(RuntimeAdStackExpected::Unknown) + } + + #[cfg(test)] + fn auction_disabled() -> Self { + Self::from_expected(RuntimeAdStackExpected::No) + } +} + +/// Confirmation status for a single configured slot (compare-side mirror of the +/// output `SlotStatus`). +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, +} + +/// The verification result for one audited page. +#[derive(Debug, Clone)] +pub struct PageVerificationResult { + /// Whether the runtime ad stack was expected to run for this page. + pub runtime_ad_stack_expected: RuntimeAdStackExpected, + /// Per-slot results, in expected-slot order. + pub slots: Vec, + /// Live evidence that matched no configured slot. + pub extra_evidence: Vec, +} + +impl PageVerificationResult { + /// Whether `--strict` should fail for this page. + /// + /// False when the runtime ad stack is not expected to run (a known gate + /// suppressed it); otherwise true if any slot is missing or partial. Provider + /// warnings and extra evidence alone never fail strict. + #[must_use] + pub fn strict_failed(&self) -> bool { + if self.runtime_ad_stack_expected == RuntimeAdStackExpected::No { + return false; + } + self.slots + .iter() + .any(|slot| matches!(slot.status, SlotStatus::Missing | SlotStatus::Partial)) + } +} + +/// Per-slot verification result. +#[derive(Debug, Clone)] +pub struct SlotResult { + /// The configured slot id. + pub id: String, + /// The confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + pub phase: EvidencePhase, + /// The live evidence observed for this slot. + pub evidence: SlotEvidence, + /// Slot-level warnings (size, provider, etc.). + pub warnings: Vec, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone)] +pub struct SlotEvidence { + /// The resolved DOM element ID, if any. + pub dom_id: Option, + /// The matched GPT slot, if any. + pub gpt: Option, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone)] +pub struct ExtraEvidence { + /// Evidence kind: `dom`, `gpt`, or `aps`. + pub kind: String, + /// The phase it was observed in. + pub phase: EvidencePhase, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes. + pub sizes: Vec<(u32, u32)>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +fn warning(code: &str, message: String) -> Warning { + Warning { + code: code.to_string(), + message, + } +} + +/// Resolves the slot root DOM element per spec §5.3. +/// +/// Exact `div_id` match first, then the first element whose ID starts with +/// `div_id`, ignoring `-container` wrappers. +fn resolve_dom<'a>(dom_ids: &'a [DomEvidence], div_id: &str) -> Option<&'a DomEvidence> { + if let Some(exact) = dom_ids.iter().find(|dom| dom.dom_id == div_id) { + return Some(exact); + } + dom_ids + .iter() + .find(|dom| dom.dom_id.starts_with(div_id) && !dom.dom_id.ends_with("-container")) +} + +/// Returns true when a GPT slot's element ID matches the resolved DOM id (or its +/// `-container`), per spec §5.4. +fn gpt_div_matches(gpt_div: &str, expected: &ExpectedSlot, resolved_dom_id: Option<&str>) -> bool { + match resolved_dom_id { + Some(dom_id) => gpt_div == dom_id || gpt_div == format!("{dom_id}-container"), + None => { + gpt_div == expected.div_id + || (gpt_div.starts_with(&expected.div_id) && !gpt_div.ends_with("-container")) + } + } +} + +fn banner_sizes(expected: &ExpectedSlot) -> Vec<(u32, u32)> { + expected + .formats + .iter() + .filter(|format| format.media_type == "banner") + .map(|format| (format.width, format.height)) + .collect() +} + +/// Compares configured expected slots against decoded browser evidence. +#[must_use] +pub fn compare_page_evidence( + expected: &[ExpectedSlot], + evidence: &BrowserAdEvidence, + gate: RuntimeGateSummary, +) -> PageVerificationResult { + let mut consumed_gpt = vec![false; evidence.gpt_slots.len()]; + let mut slots = Vec::with_capacity(expected.len()); + + for slot in expected { + let resolved = resolve_dom(&evidence.dom_ids, &slot.div_id); + let resolved_id = resolved.map(|dom| dom.dom_id.clone()); + // An unrenderable (`None`) configured path can never match live GPT + // evidence; matching on anything else would confirm the wrong unit. + let gpt_idx = slot.gam_unit_path.as_deref().and_then(|unit_path| { + evidence.gpt_slots.iter().position(|gpt| { + gpt.gam_unit_path == unit_path + && gpt_div_matches(&gpt.div_id, slot, resolved_id.as_deref()) + }) + }); + + let banner = banner_sizes(slot); + let mut warnings = Vec::new(); + if slot.gam_unit_path.is_none() { + warnings.push(warning( + "gam_unit_path_unrenderable", + format!( + "slot `{}` gam_unit_path template renders past GAM's unit-path byte limit \ + for this page's section; the runtime rejects this config", + slot.id + ), + )); + } + + let (status, dom_for_evidence, gpt_for_evidence, phase) = if let Some(idx) = gpt_idx { + consumed_gpt[idx] = true; + let gpt = &evidence.gpt_slots[idx]; + let dom_id = resolved_id.clone().or_else(|| Some(gpt.div_id.clone())); + if banner.is_empty() { + warnings.push(warning( + "unsupported_format", + format!( + "slot `{}` has only non-banner formats; not confirmable in Phase 1", + slot.id + ), + )); + (SlotStatus::Partial, dom_id, Some(gpt.clone()), gpt.phase) + } else if gpt.sizes.is_empty() { + warnings.push(warning( + "out_of_page_slot", + format!( + "slot `{}` matched an out-of-page GPT slot with no sizes", + slot.id + ), + )); + (SlotStatus::Partial, dom_id, Some(gpt.clone()), gpt.phase) + } else if banner.iter().any(|size| gpt.sizes.contains(size)) { + let extra: Vec<(u32, u32)> = gpt + .sizes + .iter() + .copied() + .filter(|size| !banner.contains(size)) + .collect(); + if !extra.is_empty() { + warnings.push(warning( + "extra_observed_size", + format!("slot `{}` observed extra GPT sizes {extra:?}", slot.id), + )); + } + let missing: Vec<(u32, u32)> = banner + .iter() + .copied() + .filter(|size| !gpt.sizes.contains(size)) + .collect(); + if !missing.is_empty() { + warnings.push(warning( + "configured_size_not_observed", + format!( + "slot `{}` configured sizes {missing:?} were not observed", + slot.id + ), + )); + } + (SlotStatus::Confirmed, dom_id, Some(gpt.clone()), gpt.phase) + } else { + warnings.push(warning( + "incompatible_sizes", + format!( + "slot `{}` GPT path and div matched but no configured size overlapped", + slot.id + ), + )); + (SlotStatus::Partial, dom_id, Some(gpt.clone()), gpt.phase) + } + } else if let Some(dom) = resolved { + warnings.push(warning( + "dom_without_gpt", + "DOM element matched, but no GPT slot evidence was observed".to_string(), + )); + ( + SlotStatus::Partial, + Some(dom.dom_id.clone()), + None, + dom.phase, + ) + } else { + (SlotStatus::Missing, None, None, EvidencePhase::InitialLoad) + }; + + if let Some(aps_slot_id) = &slot.aps_slot_id { + let matched = evidence + .aps_calls + .iter() + .any(|call| &call.slot_id == aps_slot_id); + if !matched { + warnings.push(warning( + "aps_evidence_missing", + format!("configured APS slot `{aps_slot_id}` had no fetchBids evidence"), + )); + } + } + + slots.push(SlotResult { + id: slot.id.clone(), + status, + phase, + evidence: SlotEvidence { + dom_id: dom_for_evidence, + gpt: gpt_for_evidence, + }, + warnings, + }); + } + + let extra_evidence = evidence + .gpt_slots + .iter() + .enumerate() + .filter(|(idx, _)| !consumed_gpt[*idx]) + .map(|(_, gpt)| ExtraEvidence { + kind: "gpt".to_string(), + phase: gpt.phase, + dom_id: Some(gpt.div_id.clone()), + gam_unit_path: Some(gpt.gam_unit_path.clone()), + sizes: gpt.sizes.clone(), + reason: "no_configured_slot_matched".to_string(), + }) + .collect(); + + PageVerificationResult { + runtime_ad_stack_expected: gate.expected, + slots, + extra_evidence, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::expected::ExpectedFormat; + + fn dom(id: &str) -> DomEvidence { + DomEvidence { + dom_id: id.to_string(), + phase: EvidencePhase::InitialLoad, + } + } + + fn gpt_slot(gam_unit_path: &str, div_id: &str, sizes: &[(u32, u32)]) -> GptSlotEvidence { + GptSlotEvidence { + gam_unit_path: gam_unit_path.to_string(), + div_id: div_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn aps(slot_id: &str, sizes: &[(u32, u32)]) -> ApsFetchBidsEvidence { + ApsFetchBidsEvidence { + slot_id: slot_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn evidence( + doms: Vec, + gpts: Vec, + aps: Vec, + ) -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: doms, + gpt_slots: gpts, + aps_calls: aps, + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn expected_slot( + id: &str, + div_id: &str, + gam_unit_path: &str, + sizes: &[(u32, u32)], + providers: &[&str], + ) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: sizes + .iter() + .map(|&(width, height)| ExpectedFormat { + width, + height, + media_type: "banner".to_string(), + }) + .collect(), + providers: providers.iter().copied().map(String::from).collect(), + aps_slot_id: providers.contains(&"aps").then(|| id.to_string()), + page_patterns: Vec::new(), + } + } + + fn expected_slot_video(id: &str, div_id: &str, gam_unit_path: &str) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: vec![ExpectedFormat { + width: 0, + height: 0, + media_type: "video".to_string(), + }], + providers: Vec::new(), + aps_slot_id: None, + page_patterns: Vec::new(), + } + } + + #[test] + fn gpt_path_div_and_size_overlap_confirms_slot() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + result.slots[0].warnings.is_empty(), + "confirmed slot should carry no warnings" + ); + } + + #[test] + fn unrenderable_gam_unit_path_never_confirms() { + let mut expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + expected.gam_unit_path = None; + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Partial, + "an unrenderable configured path must not confirm against GPT evidence" + ); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "gam_unit_path_unrenderable"), + "should explain why the slot cannot be confirmed" + ); + } + + #[test] + fn dom_only_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(vec![dom("ad-atf-0")], Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "dom_without_gpt") + ); + } + + #[test] + fn no_dom_or_gpt_is_missing() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Missing); + } + + #[test] + fn prefix_dom_resolution_ignores_container_suffix() { + let expected = expected_slot( + "header", + "ad-header-0-", + "/123/homepage/header", + &[(728, 90)], + &[], + ); + let evidence = evidence( + vec![dom("ad-header-0--container"), dom("ad-header-0-_R_abc123")], + Vec::new(), + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].evidence.dom_id.as_deref(), + Some("ad-header-0-_R_abc123"), + "prefix match should skip -container" + ); + assert_eq!(result.slots[0].status, SlotStatus::Partial); + } + + #[test] + fn unmatched_gpt_slot_becomes_extra_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![ + gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)]), + gpt_slot( + "/123/publisher/right-rail", + "ad-right-rail-0", + &[(300, 250)], + ), + ], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert_eq!(result.extra_evidence.len(), 1); + assert_eq!(result.extra_evidence[0].kind, "gpt"); + assert!( + !result.strict_failed(), + "extra evidence alone must not fail strict" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::auction_disabled(), + ); + + assert_eq!(result.runtime_ad_stack_expected, RuntimeAdStackExpected::No); + assert_eq!(result.slots[0].status, SlotStatus::Missing); + assert!( + !result.strict_failed(), + "missing slot must not fail strict when ad stack is No" + ); + } + + #[test] + fn gpt_incompatible_sizes_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(728, 90)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "incompatible_sizes") + ); + } + + #[test] + fn non_banner_only_slot_is_partial() { + let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); + let evidence = evidence( + vec![dom("ad-video-0")], + vec![gpt_slot("/123/news/video", "ad-video-0", &[(640, 480)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "unsupported_format") + ); + } + + #[test] + fn gpt_container_element_id_confirms() { + let expected = expected_slot("atf", "ad-atf-0", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0"), dom("ad-atf-0-container")], + vec![gpt_slot( + "/123/news/atf", + "ad-atf-0-container", + &[(300, 250)], + )], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "container element id is a valid GPT div match" + ); + } + + #[test] + fn out_of_page_gpt_slot_warns_and_does_not_confirm() { + let expected = expected_slot( + "interstitial", + "ad-oop-", + "/123/news/oop", + &[(300, 250)], + &[], + ); + let evidence = evidence( + vec![dom("ad-oop-0")], + vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_ne!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "out_of_page_slot") + ); + } + + #[test] + fn aps_match_adds_no_warning() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + vec![aps("atf", &[(300, 250)])], + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + !result.slots[0] + .warnings + .iter() + .any(|w| w.code.starts_with("aps_")), + "matching APS should not warn" + ); + } + + #[test] + fn aps_missing_warns_but_keeps_confirmed() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "missing APS does not flip status" + ); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "aps_evidence_missing") + ); + assert!( + !result.strict_failed(), + "provider warning alone must not fail strict" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs new file mode 100644 index 000000000..549ec0a57 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -0,0 +1,312 @@ +//! Pure expected-slot projection from the runtime creative-opportunity matcher. +//! +//! This module owns path/URL normalization and converts the slots matched by +//! [`match_slots`] into stable, owned [`ExpectedSlot`] records for output and +//! browser-evidence comparison. It must not duplicate glob-matching semantics. + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{CreativeOpportunitiesConfig, match_slots}; +use url::Url; + +/// The expected slots for a single page path, in configured slot order. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlots { + /// The page path the slots were matched against. + pub path: String, + /// Matched slots projected into stable records, in configured order. + pub slots: Vec, +} + +/// A single configured slot expected to appear for a page path. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlot { + /// The slot identifier. + pub id: String, + /// Resolved HTML `div` element ID (override or the slot id). + pub div_id: String, + /// Resolved GAM unit path: the rendered `gam_unit_path` template (or + /// `//` when the slot has none). + /// + /// `None` when a dynamic template renders beyond GAM's unit-path byte limit + /// for this path's section. Runtime validation rejects such a config, so + /// this only occurs for a config that would fail to load; the slot is then + /// reported unconfirmable rather than matched against a wrong path. + pub gam_unit_path: Option, + /// Configured ad formats. + pub formats: Vec, + /// Configured provider names, in `aps`, `prebid` order. + pub providers: Vec, + /// Configured APS slot ID, when the `aps` provider is set. Used to match + /// `apstag.fetchBids` evidence; not part of the §8 JSON output. + pub aps_slot_id: Option, + /// Glob patterns configured for this slot. + pub page_patterns: Vec, +} + +/// A configured ad format as a stable width/height/media-type record. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedFormat { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Media type rendered as a stable string (`banner`, `video`, `native`). + pub media_type: String, +} + +/// Projects the slots matching `path` into stable expected-slot records. +/// +/// Uses [`match_slots`] so glob semantics stay identical to the runtime, and +/// preserves configured slot order. `path` is assumed already normalized via +/// [`normalize_path_or_url`]. +/// +/// `gam_unit_path` templates are rendered against the section the runtime would +/// derive from `path` (per the config's `section_root`/`section_segment` +/// policy), so `{section}`-bearing configs project the same unit path the live +/// page requests. +// Shared projection used by the audit verifier; the static commands match slots +// directly against the runtime matcher. +#[must_use] +pub fn expected_slots_for_path(path: &str, config: &CreativeOpportunitiesConfig) -> ExpectedSlots { + let section = config.section_for_path(path); + let slots = match_slots(&config.slot, path) + .into_iter() + .map(|slot| ExpectedSlot { + id: slot.id.clone(), + div_id: slot.resolved_div_id().to_string(), + gam_unit_path: slot.render_gam_unit_path(&config.gam_network_id, §ion), + formats: slot + .formats + .iter() + .map(|format| ExpectedFormat { + width: format.width, + height: format.height, + media_type: media_type_str(&format.media_type).to_string(), + }) + .collect(), + providers: provider_names(slot), + aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), + page_patterns: slot.page_patterns.clone(), + }) + .collect(); + + ExpectedSlots { + path: path.to_string(), + slots, + } +} + +fn media_type_str(media_type: &MediaType) -> &'static str { + match media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + } +} + +fn provider_names( + slot: &trusted_server_core::creative_opportunities::CreativeOpportunitySlot, +) -> Vec { + let mut providers = Vec::new(); + if slot.providers.aps.is_some() { + providers.push("aps".to_string()); + } + if slot.providers.prebid.is_some() { + providers.push("prebid".to_string()); + } + providers +} + +/// Normalizes a page path or full URL into a request path. +/// +/// Full `scheme://` inputs are parsed and reduced to their path; bare inputs have +/// query and fragment stripped and a leading `/` ensured. Empty paths become `/`. +/// +/// # Errors +/// +/// Returns a user-facing string when a `scheme://` input cannot be parsed as a URL. +pub fn normalize_path_or_url(input: &str) -> Result { + if input.contains("://") { + let url = Url::parse(input).map_err(|err| format!("invalid URL `{input}`: {err}"))?; + let path = url.path(); + return Ok(if path.is_empty() { + "/".to_string() + } else { + path.to_string() + }); + } + + let without_fragment = input.split('#').next().unwrap_or(input); + let path = without_fragment + .split('?') + .next() + .unwrap_or(without_fragment); + if path.is_empty() { + Ok("/".to_string()) + } else if path.starts_with('/') { + Ok(path.to_string()) + } else { + Ok(format!("/{path}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn creative_config_with_slots(patterns: &[&str]) -> CreativeOpportunitiesConfig { + let page_patterns = patterns + .iter() + .map(|pattern| format!("\"{pattern}\"")) + .collect::>() + .join(", "); + let toml = format!( + "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [{page_patterns}]\n\ + formats = [{{ width = 300, height = 250 }}]\n\ + \n\ + [slot.providers.prebid]\n\ + bidders = {{}}\n" + ); + let mut config = toml::from_str::(&toml) + .expect("should deserialize creative opportunities config"); + config.compile_slots(); + config + } + + #[test] + fn expected_slots_use_runtime_matcher_and_config_order() { + let config = creative_config_with_slots(&["/news/*", "/"]); + let expected = expected_slots_for_path("/news/story", &config); + + assert_eq!(expected.path, "/news/story"); + assert_eq!( + expected + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(), + ["atf"] + ); + assert_eq!(expected.slots[0].div_id, "ad-atf-"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/123/news/atf") + ); + assert_eq!(expected.slots[0].providers, ["prebid"]); + assert_eq!( + expected.slots[0].formats, + vec![ExpectedFormat { + width: 300, + height: 250, + media_type: "banner".to_string(), + }] + ); + } + + #[test] + fn expected_slots_default_resolution_without_overrides() { + let toml = "gam_network_id = \"42\"\n\ + \n\ + [[slot]]\n\ + id = \"footer\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let expected = expected_slots_for_path("/", &config); + assert_eq!(expected.slots[0].div_id, "footer"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/42/footer") + ); + assert!(expected.slots[0].providers.is_empty()); + } + + #[test] + fn expected_slots_render_section_templates_per_path() { + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\", \"/news\", \"/news/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + // A path with a section segment renders that segment. + assert_eq!( + expected_slots_for_path("/news/story", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/news"), + "a section template should render the path's section" + ); + // The site root falls back to the configured section_root. + assert_eq!( + expected_slots_for_path("/", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/homepage"), + "the root path should render section_root" + ); + } + + #[test] + fn expected_slots_report_unrenderable_dynamic_template_as_none() { + // A `{section}` template that renders past GAM's 100-byte unit-path + // limit. `validate_runtime` rejects this config, so the verifier reports + // the slot as unconfirmable rather than matching a truncated path. + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{section}/{section}\"\n\ + page_patterns = [\"/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let long_path = format!("/{}", "a".repeat(60)); + let expected = expected_slots_for_path(&long_path, &config); + + assert_eq!( + expected.slots[0].gam_unit_path, None, + "an over-limit dynamic render should project as None" + ); + } + + #[test] + fn normalize_path_or_url_strips_query_and_fragment() { + assert_eq!( + normalize_path_or_url("https://www.example.com/news/story?x=1#top") + .expect("should normalize"), + "/news/story" + ); + assert_eq!( + normalize_path_or_url("news/story?x=1").expect("should normalize"), + "/news/story" + ); + } + + #[test] + fn normalize_path_or_url_roots_empty_input() { + assert_eq!( + normalize_path_or_url("https://www.example.com").expect("should normalize"), + "/" + ); + assert_eq!(normalize_path_or_url("").expect("should normalize"), "/"); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/mod.rs b/crates/trusted-server-cli/src/ad_templates/mod.rs new file mode 100644 index 000000000..3c26bf121 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/mod.rs @@ -0,0 +1,7 @@ +//! Pure, host-only ad-template CLI logic shared by the static `ts config +//! ad-templates ...` commands and the browser-backed `ts audit ad-templates +//! verify` command. + +pub mod compare; +pub mod expected; +pub mod output; diff --git a/crates/trusted-server-cli/src/ad_templates/output.rs b/crates/trusted-server-cli/src/ad_templates/output.rs new file mode 100644 index 000000000..9a1190652 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/output.rs @@ -0,0 +1,444 @@ +//! Stable, serializable output model for ad-template diagnostics. +//! +//! These types mirror the `--json` contract in +//! `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` §8. +//! Field names and declaration order are load-bearing: `serde` serializes struct +//! fields in declaration order, so the order here must match the spec examples. +//! +//! The model is consumed by the `ts audit ad-templates verify` orchestrator +//! (Task 9), which assembles these wire types from the URL/gate context and the +//! pure comparison result. Until that consumer lands, the types are exercised only +//! by tests, hence the module-scoped `dead_code` allow. +#![allow( + dead_code, + reason = "wire model assembled by the audit verifier in a later task" +)] + +use std::borrow::Cow; + +use serde::{Deserialize, Serialize}; + +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +/// Escapes control characters in page-controlled text bound for a terminal. +/// +/// Page titles and collector warning messages are attacker-controlled: an +/// audited page can put ANSI/OSC escape sequences in `document.title` and drive +/// the operator's terminal (cursor movement, clipboard writes, forged output) +/// when the value is printed verbatim. Every C0 control (including ESC), DEL, +/// and the C1 range are rendered as `\u{XXXX}` so the text stays inert. JSON +/// output is unaffected — `serde_json` escapes these already. +/// +/// Returns a borrowed `Cow` when the input needs no escaping. +#[must_use] +pub fn escape_terminal_text(value: &str) -> Cow<'_, str> { + if !value.chars().any(is_terminal_control) { + return Cow::Borrowed(value); + } + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + if is_terminal_control(ch) { + escaped.push_str(&format!("\\u{{{:04X}}}", ch as u32)); + } else { + escaped.push(ch); + } + } + Cow::Owned(escaped) +} + +/// Whether `ch` can act as a terminal control code (C0, DEL, or C1). +fn is_terminal_control(ch: char) -> bool { + let code = ch as u32; + code < 0x20 || (0x7f..=0x9f).contains(&code) +} + +/// Confirmation status for a single configured slot. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, +} + +/// JSON rendering of the runtime ad-stack expectation. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeAdStackExpectedJson { + /// The server-side ad stack is expected to run. + Yes, + /// A known gate blocks the server-side ad stack. + No, + /// Consent or another gate is unprovable. + Unknown, +} + +impl From for RuntimeAdStackExpectedJson { + fn from(value: RuntimeAdStackExpected) -> Self { + match value { + RuntimeAdStackExpected::Yes => Self::Yes, + RuntimeAdStackExpected::No => Self::No, + RuntimeAdStackExpected::Unknown => Self::Unknown, + } + } +} + +/// State of a single runtime gate. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GateState { + /// The gate passed. + Pass, + /// The gate blocked the ad stack. + Fail, + /// The gate state could not be proven. + Unknown, +} + +/// Evidence-collection phase, rendered for JSON output. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhaseJson { + /// Observed during the initial page load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A structured warning with a stable machine code and human message. +/// +/// `Serialize` for output; `Deserialize` because the browser collector payload +/// carries warning objects decoded into the comparison input. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct Warning { + /// Stable machine-readable code (e.g. `dom_without_gpt`). + pub code: String, + /// Human-readable message; JSON consumers must not parse this. + pub message: String, +} + +/// Top-level `--json` document for `ts audit ad-templates verify`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct VerificationReport { + /// True when no strict failure and no page-level error occurred. + pub ok: bool, + /// Whether `--strict` was set. + pub strict: bool, + /// One entry per requested URL, in input order. + pub pages: Vec, + /// Run-level warnings not attributable to a single page. + pub warnings: Vec, +} + +/// A single audited page result. +/// +/// `error` is declared immediately after `path` so the serialized key order +/// matches the spec §8 `navigation_failed` shape; on normal pages it is `None` +/// and skipped, leaving the runtime/gates fields in §8 order. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PageJson { + /// The requested URL. + pub url: String, + /// The final URL after redirects, or `null` on navigation failure. + pub final_url: Option, + /// The requested URL's path. + pub requested_path: String, + /// The final path used for matching, or `null` on navigation failure. + pub path: Option, + /// Present only on a page-level collection failure. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Three-state runtime ad-stack expectation; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_ad_stack_expected: Option, + /// Per-gate evidence; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub gates: Option, + /// Number of configured slots matched for the final path; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_slot_count: Option, + /// Per-slot verification results. + pub slots: Vec, + /// Live ad-slot evidence with no matching configured slot. + pub extra_evidence: Vec, + /// Page-level warnings. + pub warnings: Vec, +} + +/// Runtime gate states for a page, one field per spec §5.2 gate. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Gates { + /// Request method is `GET`. + pub method_get: GateState, + /// Request is a top-level navigation. + pub navigation: GateState, + /// Request is not a prefetch. + pub not_prefetch: GateState, + /// Request is not from a known bot. + pub not_bot: GateState, + /// At least one configured slot matched the final path. + pub matched_slots: GateState, + /// The `[auction].enabled` kill switch is on. + pub auction_enabled: GateState, + /// Consent allows the auction (often `unknown` for live requests). + pub consent_allows_auction: GateState, +} + +/// A single configured slot's verification result. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotJson { + /// The configured slot id. + pub id: String, + /// The slot's confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + pub phase: EvidencePhaseJson, + /// The configured shape of the slot (no `id`/`page_patterns` per §8). + pub configured: ConfiguredJson, + /// The live evidence observed for this slot. + pub evidence: SlotEvidenceJson, + /// Slot-level warnings (e.g. provider or size warnings). + pub warnings: Vec, +} + +/// The configured shape of a slot, as rendered in §8 `configured`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ConfiguredJson { + /// Resolved div element ID. + pub div_id: String, + /// Resolved GAM unit path, or `null` when a dynamic template renders past + /// GAM's unit-path byte limit for this page's section. + pub gam_unit_path: Option, + /// Configured formats. + pub formats: Vec, + /// Configured provider names. + pub providers: Vec, +} + +/// A configured format, as rendered in §8. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct FormatJson { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Media type string (`banner`, `video`, `native`). + pub media_type: String, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotEvidenceJson { + /// The resolved DOM element ID observed, if any. + pub dom_id: Option, + /// GPT slot evidence, if any (no `phase` key per §8). + pub gpt: Option, +} + +/// GPT slot evidence, as rendered in §8 `evidence.gpt`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct GptEvidenceJson { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ExtraEvidenceJson { + /// Evidence kind: `dom`, `gpt`, or `aps`. + pub kind: String, + /// The phase the evidence was observed in. + pub phase: EvidencePhaseJson, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +#[cfg(test)] +impl VerificationReport { + fn example_confirmed_with_extra_evidence() -> Self { + VerificationReport { + ok: true, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/news/story".to_string(), + final_url: Some("https://www.example.com/news/story".to_string()), + requested_path: "/news/story".to_string(), + path: Some("/news/story".to_string()), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::Unknown), + gates: Some(Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: GateState::Pass, + auction_enabled: GateState::Pass, + consent_allows_auction: GateState::Unknown, + }), + matched_slot_count: Some(1), + slots: vec![SlotJson { + id: "atf".to_string(), + status: SlotStatus::Confirmed, + phase: EvidencePhaseJson::InitialLoad, + configured: ConfiguredJson { + div_id: "ad-atf-".to_string(), + gam_unit_path: Some("/123/news/atf".to_string()), + formats: vec![FormatJson { + width: 300, + height: 250, + media_type: "banner".to_string(), + }], + providers: vec!["aps".to_string()], + }, + evidence: SlotEvidenceJson { + dom_id: Some("ad-atf-0".to_string()), + gpt: Some(GptEvidenceJson { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![[300, 250]], + }), + }, + warnings: Vec::new(), + }], + extra_evidence: vec![ExtraEvidenceJson { + kind: "gpt".to_string(), + phase: EvidencePhaseJson::InitialLoad, + dom_id: Some("ad-right-rail-0".to_string()), + gam_unit_path: Some("/123/publisher/right-rail".to_string()), + sizes: vec![[300, 250]], + reason: "no_configured_slot_matched".to_string(), + }], + warnings: vec![Warning { + code: "redirected".to_string(), + message: "navigation redirected to the final path".to_string(), + }], + }], + warnings: Vec::new(), + } + } + + fn example_navigation_failed() -> Self { + VerificationReport { + ok: false, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/broken".to_string(), + final_url: None, + requested_path: "/broken".to_string(), + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: "failed to read main document navigation response".to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + }], + warnings: Vec::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escape_terminal_text_passes_through_ordinary_titles() { + assert!( + matches!( + escape_terminal_text("Example News — Story"), + Cow::Borrowed(_) + ), + "text with no control characters should not allocate" + ); + assert_eq!( + escape_terminal_text("Example News — Story"), + "Example News — Story" + ); + } + + #[test] + fn escape_terminal_text_neutralizes_control_sequences() { + // ESC-based CSI/OSC sequences and a raw newline are the terminal-driving + // primitives a hostile page would put in `document.title`. + assert_eq!( + escape_terminal_text("a\u{1b}]0;pwned\u{7}b"), + "a\\u{001B}]0;pwned\\u{0007}b", + "ESC and BEL should be rendered inert" + ); + assert_eq!( + escape_terminal_text("line\nforged: ok"), + "line\\u{000A}forged: ok", + "a newline should not let a title forge an output line" + ); + assert_eq!( + escape_terminal_text("del\u{7f}c1\u{9b}"), + "del\\u{007F}c1\\u{009B}", + "DEL and the C1 range should be escaped too" + ); + } + + #[test] + fn verification_json_contains_gate_state_and_extra_evidence() { + let result = VerificationReport::example_confirmed_with_extra_evidence(); + let value = serde_json::to_value(&result).expect("should serialize"); + + assert_eq!(value["ok"], true); + assert_eq!(value["pages"][0]["requested_path"], "/news/story"); + assert_eq!(value["pages"][0]["runtime_ad_stack_expected"], "unknown"); + assert_eq!( + value["pages"][0]["gates"]["consent_allows_auction"], + "unknown" + ); + assert_eq!(value["pages"][0]["slots"][0]["status"], "confirmed"); + assert_eq!( + value["pages"][0]["slots"][0]["evidence"]["gpt"]["sizes"][0][0], + 300 + ); + assert_eq!(value["pages"][0]["extra_evidence"][0]["kind"], "gpt"); + assert_eq!(value["pages"][0]["warnings"][0]["code"], "redirected"); + // `configured` excludes id/page_patterns per §8. + assert!(value["pages"][0]["slots"][0]["configured"]["id"].is_null()); + assert!(value["pages"][0]["slots"][0]["configured"]["page_patterns"].is_null()); + // `evidence.gpt` has no `phase` key per §8. + assert!(value["pages"][0]["slots"][0]["evidence"]["gpt"]["phase"].is_null()); + } + + #[test] + fn page_error_json_matches_navigation_failed_shape() { + let result = VerificationReport::example_navigation_failed(); + let value = serde_json::to_value(&result).expect("should serialize"); + let page = &value["pages"][0]; + + assert_eq!(page["error"]["code"], "navigation_failed"); + assert!(page["final_url"].is_null(), "final_url should be null"); + assert!(page["path"].is_null(), "path should be null"); + assert!( + page.get("runtime_ad_stack_expected").is_none(), + "runtime field absent on error page" + ); + assert!(page.get("gates").is_none(), "gates absent on error page"); + assert!( + page.get("matched_slot_count").is_none(), + "matched_slot_count absent on error page" + ); + assert_eq!(value["ok"], false); + } +} diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs new file mode 100644 index 000000000..fadf58cde --- /dev/null +++ b/crates/trusted-server-cli/src/app_config.rs @@ -0,0 +1,146 @@ +//! Shared effective Trusted Server app-config loading for the `ts` CLI. +//! +//! Both the static `ts config ad-templates ...` commands and the browser-backed +//! `ts audit ad-templates verify` command load the same effective app config +//! through [`load_settings`], so config-path resolution and the `EdgeZero` +//! environment overlay stay consistent across command families. + +use std::path::{Path, PathBuf}; + +use clap::Args; +use edgezero_core::app_config::{self, AppConfigLoadOptions}; +use edgezero_core::manifest::ManifestLoader; +use trusted_server_core::config::TrustedServerAppConfig; +use trusted_server_core::settings::Settings; + +/// Shared local app-config flags accepted by every config/audit ad-template command. +#[derive(Clone, Debug, Args)] +pub struct AppConfigArgs { + /// Path to `trusted-server.toml`. Defaults to `.toml` beside `edgezero.toml`. + #[arg(long)] + pub app_config: Option, + /// Path to `edgezero.toml`. + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + /// Skip app-config environment overlay. + #[arg(long)] + pub no_env: bool, +} + +/// Effective settings plus the resolved app-config path they were loaded from. +#[derive(Debug)] +pub struct LoadedSettings { + /// The `trusted-server.toml` path the settings were loaded from. + pub app_config_path: PathBuf, + /// The deserialized effective settings. + pub settings: Settings, +} + +/// Loads the effective Trusted Server settings described by `args`. +/// +/// Resolves the app-config path from `args` (or the manifest's `.toml` +/// default), applies the `EdgeZero` environment overlay unless `no_env` is set, and +/// returns the deserialized [`Settings`]. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded, has no +/// `[app].name`, or the resolved app-config file cannot be read or parsed. When an +/// explicit `--app-config` path is given and is missing, the error names that +/// exact path rather than silently falling back. +pub fn load_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, !args.no_env) +} + +/// Loads Trusted Server settings from the resolved app-config file without +/// applying environment overlays. +/// +/// Mutating commands use this path so environment-only values are never +/// persisted into the operator-owned TOML file. +/// +/// # Errors +/// +/// Returns the same path-resolution, read, and parse errors as +/// [`load_settings`]. +pub fn load_file_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, false) +} + +fn load_settings_with_env_overlay( + args: &AppConfigArgs, + env_overlay: bool, +) -> Result { + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + let app_config_path = + resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); + + let mut opts = AppConfigLoadOptions::default(); + opts.env_overlay = env_overlay; + let app_config = app_config::deserialize_app_config_with_options::( + &app_config_path, + &app_name, + &opts, + ) + .map_err(|err| format!("failed to load {}: {err}", app_config_path.display()))?; + + Ok(LoadedSettings { + app_config_path, + settings: app_config.into_settings(), + }) +} + +fn resolve_app_config_path( + explicit: Option<&Path>, + manifest_path: &Path, + app_name: &str, +) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + let file_name = format!("{app_name}.toml"); + if let Some(parent) = manifest_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + parent.join(file_name) + } else { + PathBuf::from(file_name) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + #[test] + fn explicit_missing_app_config_does_not_fall_back() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let missing_path = temp.path().join("missing.toml"); + + let args = AppConfigArgs { + app_config: Some(missing_path.clone()), + manifest: manifest_path, + no_env: true, + }; + + let err = load_settings(&args).expect_err("should reject missing explicit config"); + assert!( + err.contains(missing_path.to_string_lossy().as_ref()), + "error should mention the explicit missing path" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js new file mode 100644 index 000000000..1133d46b6 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -0,0 +1,199 @@ +// Read-only ad-template evidence collector, injected before publisher scripts run. +// +// This body runs inside an IIFE that defines `__TS_CONFIG` (configured div +// prefixes + APS slot IDs). It records evidence into `window.__tsAdTemplateEvidence` +// and never captures page HTML, cookies, storage, request bodies, or arbitrary DOM. +// It always calls original page functions with unchanged arguments and never +// spoofs the browser automation flag. + +const __ts_config = typeof __TS_CONFIG === "object" && __TS_CONFIG ? __TS_CONFIG : {} +const __ts_prefixes = Array.isArray(__ts_config.div_prefixes) ? __ts_config.div_prefixes : [] + +const __ts_ev = (window.__tsAdTemplateEvidence = window.__tsAdTemplateEvidence || { + dom_ids: [], + gpt_slots: [], + aps_calls: [], + warnings: [], +}) + +const __ts_phase = () => (window.__tsScrollPhase ? "scroll" : "initial_load") + +// Hard cap per evidence list so a hostile page cannot grow the store without +// bound; the page controls how many slots/elements/warnings it produces. +const __ts_max_entries = 1024 +function __ts_push(list, entry) { + if (list.length < __ts_max_entries) list.push(entry) +} + +// GPT sizes reach Rust as u32 pairs, so anything non-integral (fluid slots, +// NaN, negative or fractional dimensions) must be dropped here — a single bad +// pair would fail deserialization of the whole evidence payload and discard +// every other slot's otherwise valid evidence. +function __ts_size_pair(width, height) { + if (!Number.isInteger(width) || !Number.isInteger(height)) return null + if (width < 0 || height < 0) return null + return [width, height] +} + +function __ts_normalize_sizes(sizes) { + const out = [] + if (!Array.isArray(sizes)) return out + // Accept [w, h] or [[w, h], ...]; treat numeric-leading arrays as a single pair. + const pairs = typeof sizes[0] === "number" ? [sizes] : sizes + for (const size of pairs) { + if (out.length >= __ts_max_entries) break + const pair = Array.isArray(size) ? __ts_size_pair(size[0], size[1]) : null + if (pair) { + out.push(pair) + } else { + __ts_push(__ts_ev.warnings, { + code: "fluid_size_ignored", + message: "non-numeric GPT size ignored", + }) + } + } + return out +} + +function __ts_record_define_slot(adUnitPath, sizes, divId) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: String(adUnitPath), + div_id: String(divId), + sizes: __ts_normalize_sizes(sizes), + phase: __ts_phase(), + }) +} + +function __ts_wrap_googletag(googletag) { + if (!googletag || googletag.__tsWrapped) return googletag + googletag.__tsWrapped = true + googletag.cmd = googletag.cmd || [] + // Wrap cmd.push without changing callback order (pass-through to the original). + const originalPush = googletag.cmd.push.bind(googletag.cmd) + googletag.cmd.push = function (callback) { + return originalPush(callback) + } + // Wrap defineSlot so both direct calls and calls dispatched from the cmd queue + // are recorded (queued callbacks call this same wrapped function). + const originalDefineSlot = googletag.defineSlot + if (typeof originalDefineSlot === "function") { + googletag.defineSlot = function (adUnitPath, sizes, divId) { + const slot = originalDefineSlot.apply(this, arguments) + try { + __ts_record_define_slot(adUnitPath, sizes, divId) + } catch (error) { + __ts_push(__ts_ev.warnings, { code: "define_slot_capture_failed", message: String(error) }) + } + return slot + } + } + return googletag +} + +function __ts_wrap_apstag(apstag) { + if (!apstag || apstag.__tsWrapped) return apstag + apstag.__tsWrapped = true + const originalFetchBids = apstag.fetchBids + if (typeof originalFetchBids === "function") { + apstag.fetchBids = function (config, callback) { + try { + const slots = (config && config.slots) || [] + for (const slot of slots) { + __ts_push(__ts_ev.aps_calls, { + slot_id: String(slot.slotID || slot.slotName || ""), + sizes: __ts_normalize_sizes(slot.sizes), + phase: __ts_phase(), + }) + } + } catch (error) { + __ts_push(__ts_ev.warnings, { code: "aps_capture_failed", message: String(error) }) + } + return originalFetchBids.apply(this, arguments) + } + } + return apstag +} + +// Wrap an existing global or intercept a later assignment of it. +function __ts_install(name, wrap) { + if (window[name]) { + wrap(window[name]) + return + } + let internal + Object.defineProperty(window, name, { + configurable: true, + get() { + return internal + }, + set(value) { + internal = wrap(value) + }, + }) +} + +__ts_install("googletag", __ts_wrap_googletag) +__ts_install("apstag", __ts_wrap_apstag) + +// On-demand DOM + getSlots scrape, invoked by the collector after settle/scroll. +window.__tsCollectAdTemplateEvidence = function () { + try { + const seen = new Set(__ts_ev.dom_ids.map((entry) => entry.dom_id)) + for (const element of document.querySelectorAll("[id]")) { + const id = element.id + if (id.endsWith("-container")) continue + if (__ts_prefixes.some((prefix) => id.startsWith(prefix)) && !seen.has(id)) { + __ts_push(__ts_ev.dom_ids, { dom_id: id, phase: __ts_phase() }) + seen.add(id) + } + } + const googletag = window.googletag + if (googletag && typeof googletag.pubads === "function") { + const pubads = googletag.pubads() + const slots = typeof pubads.getSlots === "function" ? pubads.getSlots() : [] + for (const slot of slots) { + try { + const path = typeof slot.getAdUnitPath === "function" ? slot.getAdUnitPath() : "" + const divId = typeof slot.getSlotElementId === "function" ? slot.getSlotElementId() : "" + const rawSizes = typeof slot.getSizes === "function" ? slot.getSizes() : [] + const sizes = [] + for (const size of rawSizes) { + if (sizes.length >= __ts_max_entries) break + let pair = null + if (size && typeof size.getWidth === "function" && typeof size.getHeight === "function") { + // A fluid GPT size answers getWidth()/getHeight() with a + // non-numeric value rather than throwing. + pair = __ts_size_pair(size.getWidth(), size.getHeight()) + } else if (Array.isArray(size)) { + pair = __ts_size_pair(size[0], size[1]) + } + if (pair) { + sizes.push(pair) + } else { + __ts_push(__ts_ev.warnings, { + code: "fluid_size_ignored", + message: "non-numeric GPT size ignored", + }) + } + } + const exists = __ts_ev.gpt_slots.some( + (entry) => entry.gam_unit_path === String(path) && entry.div_id === String(divId) + ) + if (!exists) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: String(path), + div_id: String(divId), + sizes, + phase: __ts_phase(), + }) + } + } catch (error) { + __ts_push(__ts_ev.warnings, { code: "gpt_scrape_failed", message: String(error) }) + } + } + } + } catch (error) { + __ts_push(__ts_ev.warnings, { code: "collect_failed", message: String(error) }) + } + return __ts_ev +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs new file mode 100644 index 000000000..8250e1cde --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -0,0 +1,739 @@ +//! Browser-backed `ts audit ad-templates verify` orchestration. +//! +//! For each URL: collect live evidence through an [`AuditCollector`], match +//! configured slots against the **final** (post-redirect) path, evaluate the +//! runtime gate, compare evidence, and assemble the stable §8 wire result. The +//! orchestration is collector-agnostic so it is fully tested with an in-memory +//! fake collector, with no Chrome dependency. + +use std::io::{self, Write}; + +use trusted_server_core::creative_opportunities::{ + AdStackGateInput, CreativeOpportunitiesConfig, evaluate_ad_stack_gate, +}; + +use crate::ad_templates::compare::{ + BrowserAdEvidence, EvidencePhase, ExtraEvidence, RuntimeGateSummary, SlotEvidence, SlotResult, + SlotStatus as CompareStatus, compare_page_evidence, +}; +use crate::ad_templates::expected::{ExpectedSlot, expected_slots_for_path, normalize_path_or_url}; +use crate::ad_templates::output::{ + ConfiguredJson, EvidencePhaseJson, ExtraEvidenceJson, FormatJson, GateState, Gates, + GptEvidenceJson, PageJson, RuntimeAdStackExpectedJson, SlotEvidenceJson, SlotJson, SlotStatus, + VerificationReport, Warning, escape_terminal_text, +}; +use crate::commands::audit::AuditAdTemplatesVerifyArgs; +use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, AuditCollector, BrowserCollectRequest, build_ad_template_init_script, +}; + +/// Verifies configured ad-template slots against live page evidence. +/// +/// # Errors +/// +/// Returns a user-facing string when config loading fails, or when verification +/// surfaces a page-level error or a `--strict` failure (after writing output). +pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result<(), String> { + let loaded = crate::app_config::load_settings(&args.config)?; + let collector = crate::commands::audit::browser::BrowserCollector::from_opts(&args.browser); + let report = build_report( + &collector, + loaded.settings.creative_opportunities.as_ref(), + loaded.settings.auction.enabled, + &args.urls, + VerifyOptions { + strict: args.strict, + scroll: args.scroll, + allow_cross_origin_redirect: args.allow_cross_origin_redirect, + }, + &args.cookies, + ); + + let stdout = io::stdout(); + let mut out = stdout.lock(); + if args.json { + write_json(&mut out, &report)?; + } else { + write_human(&mut out, &report)?; + } + + if report.ok { + Ok(()) + } else { + Err("ad-template verification reported problems".to_string()) + } +} + +/// Run-level verification switches. +#[derive(Debug, Clone, Copy)] +struct VerifyOptions { + /// Exit non-zero when a matched slot is missing or only partially confirmed. + strict: bool, + /// Perform a deterministic scroll pass after the initial settle. + scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + allow_cross_origin_redirect: bool, +} + +/// Builds the verification report for `urls` using `collector`. +/// +/// `creative` is the effective `[creative_opportunities]` config (if any) and +/// `auction_enabled` is the `[auction].enabled` kill switch. +fn build_report( + collector: &dyn AuditCollector, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, + urls: &[url::Url], + options: VerifyOptions, + cookies: &[(String, String)], +) -> VerificationReport { + let init_script = build_init_script(creative); + + let mut pages = Vec::with_capacity(urls.len()); + let mut any_error = false; + let mut any_strict_fail = false; + + for url in urls { + let request = BrowserCollectRequest { + url: url.clone(), + init_scripts: init_script.clone().into_iter().collect(), + scroll: options.scroll, + collect_ad_evidence: true, + cookies: cookies.to_vec(), + }; + + match collector.collect_page(request) { + Err(message) => { + any_error = true; + pages.push(error_page(url, &message)); + } + // Slots are matched on the *final* path, so a redirect to a + // different origin would let an unrelated site's evidence satisfy + // `--strict` — and the path-equality redirect warning would not even + // fire when the paths happen to agree. Reject unless opted in. + Ok(collected) + if !options.allow_cross_origin_redirect + && origin_changed(url, &collected.final_url) => + { + any_error = true; + pages.push(cross_origin_page(url, &collected.final_url)); + } + Ok(collected) => { + let (page, strict_failed) = build_page(url, &collected, creative, auction_enabled); + if options.strict && strict_failed { + any_strict_fail = true; + } + pages.push(page); + } + } + } + + let ok = !(any_error || (options.strict && any_strict_fail)); + VerificationReport { + ok, + strict: options.strict, + pages, + warnings: Vec::new(), + } +} + +/// Whether navigation left the requested URL's origin (scheme, host, or port). +fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { + requested.origin() != final_url.origin() +} + +/// Builds the read-only collector init script from the configured slots. +fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Option { + let config = AdTemplateCollectorConfig { + div_prefixes: creative + .map(|creative| { + creative + .slot + .iter() + .map(|slot| slot.resolved_div_id().to_string()) + .collect() + }) + .unwrap_or_default(), + aps_slot_ids: creative + .map(|creative| { + creative + .slot + .iter() + .filter_map(|slot| slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone())) + .collect() + }) + .unwrap_or_default(), + }; + build_ad_template_init_script(&config).ok() +} + +/// Assembles a successful page result, returning the wire `PageJson` and whether +/// the page would fail `--strict`. +fn build_page( + requested: &url::Url, + collected: &crate::commands::audit::collector::CollectedPage, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, +) -> (PageJson, bool) { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + let final_url = &collected.final_url; + let final_path = normalize_path_or_url(final_url.as_str()).unwrap_or_else(|_| "/".into()); + + let expected = creative + .map(|creative| expected_slots_for_path(&final_path, creative).slots) + .unwrap_or_default(); + let matched = !expected.is_empty(); + + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: matched, + consent_allows_auction: None, + auction_enabled, + }); + + let evidence = collected.ad_evidence.clone().unwrap_or_else(empty_evidence); + let result = compare_page_evidence( + &expected, + &evidence, + RuntimeGateSummary::from_expected(gate.expected), + ); + let strict_failed = result.strict_failed(); + + let mut warnings: Vec = collected.warnings.to_vec(); + if requested_path != final_path { + warnings.push(Warning { + code: "redirected".to_string(), + message: format!("navigation redirected from {requested_path} to {final_path}"), + }); + } + + let slots = expected + .iter() + .zip(result.slots.iter()) + .map(|(expected_slot, slot_result)| to_slot_json(expected_slot, slot_result)) + .collect(); + let extra_evidence = result.extra_evidence.iter().map(to_extra_json).collect(); + + let page = PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: Some(final_path), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::from( + result.runtime_ad_stack_expected, + )), + gates: Some(to_gates(matched, auction_enabled)), + matched_slot_count: Some(expected.len()), + slots, + extra_evidence, + warnings, + }; + (page, strict_failed) +} + +/// Builds a page-level navigation-failure result (spec §8 `navigation_failed`). +fn error_page(requested: &url::Url, message: &str) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: None, + requested_path, + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: message.to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +/// Builds a page-level cross-origin-redirect refusal. +/// +/// The final URL is reported so the operator can re-run against it explicitly +/// (or pass `--allow-cross-origin-redirect`) once they have confirmed it is +/// their own property. +fn cross_origin_page(requested: &url::Url, final_url: &url::Url) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: None, + error: Some(Warning { + code: "cross_origin_redirect".to_string(), + message: format!( + "navigation left the requested origin ({} -> {}); \ + evidence from another origin is not accepted as verification. \ + Re-run against the final URL, or pass --allow-cross-origin-redirect", + requested.origin().ascii_serialization(), + final_url.origin().ascii_serialization(), + ), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +fn empty_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: Vec::new(), + gpt_slots: Vec::new(), + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } +} + +fn to_gates(matched: bool, auction_enabled: bool) -> Gates { + let pass_if = |cond: bool| { + if cond { + GateState::Pass + } else { + GateState::Fail + } + }; + Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: pass_if(matched), + auction_enabled: pass_if(auction_enabled), + // Live consent is not provable from a browser navigation in Phase 1. + consent_allows_auction: GateState::Unknown, + } +} + +fn to_slot_json(expected: &ExpectedSlot, result: &SlotResult) -> SlotJson { + SlotJson { + id: result.id.clone(), + status: to_status(result.status), + phase: to_phase(result.phase), + configured: ConfiguredJson { + div_id: expected.div_id.clone(), + gam_unit_path: expected.gam_unit_path.clone(), + formats: expected + .formats + .iter() + .map(|format| FormatJson { + width: format.width, + height: format.height, + media_type: format.media_type.clone(), + }) + .collect(), + providers: expected.providers.clone(), + }, + evidence: to_slot_evidence(&result.evidence), + warnings: result.warnings.clone(), + } +} + +fn to_slot_evidence(evidence: &SlotEvidence) -> SlotEvidenceJson { + SlotEvidenceJson { + dom_id: evidence.dom_id.clone(), + gpt: evidence.gpt.as_ref().map(|gpt| GptEvidenceJson { + gam_unit_path: gpt.gam_unit_path.clone(), + div_id: gpt.div_id.clone(), + sizes: gpt.sizes.iter().map(|&(w, h)| [w, h]).collect(), + }), + } +} + +fn to_extra_json(extra: &ExtraEvidence) -> ExtraEvidenceJson { + ExtraEvidenceJson { + kind: extra.kind.clone(), + phase: to_phase(extra.phase), + dom_id: extra.dom_id.clone(), + gam_unit_path: extra.gam_unit_path.clone(), + sizes: extra.sizes.iter().map(|&(w, h)| [w, h]).collect(), + reason: extra.reason.clone(), + } +} + +fn to_status(status: CompareStatus) -> SlotStatus { + match status { + CompareStatus::Confirmed => SlotStatus::Confirmed, + CompareStatus::Partial => SlotStatus::Partial, + CompareStatus::Missing => SlotStatus::Missing, + } +} + +fn to_phase(phase: EvidencePhase) -> EvidencePhaseJson { + match phase { + EvidencePhase::InitialLoad => EvidencePhaseJson::InitialLoad, + EvidencePhase::Scroll => EvidencePhaseJson::Scroll, + } +} + +fn write_json(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + let json = serde_json::to_string_pretty(report) + .map_err(|error| format!("failed to serialize verification report: {error}"))?; + writeln!(out, "{json}").map_err(write_err) +} + +fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + // Warning codes and messages can originate in the audited page (the + // collector forwards `String(error)` from page scripts), so escape control + // characters before writing them to the operator's terminal. + let write_warning = |out: &mut dyn Write, indent: &str, warning: &Warning| { + writeln!( + out, + "{indent}warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(write_err) + }; + + for page in &report.pages { + writeln!(out, "url: {}", page.url).map_err(write_err)?; + if let Some(error) = &page.error { + writeln!( + out, + " error [{}]: {}", + escape_terminal_text(&error.code), + escape_terminal_text(&error.message) + ) + .map_err(write_err)?; + continue; + } + if let Some(path) = &page.path { + writeln!(out, " path: {path}").map_err(write_err)?; + } + for slot in &page.slots { + writeln!(out, " slot {}: {}", slot.id, status_label(slot.status)) + .map_err(write_err)?; + for warning in &slot.warnings { + write_warning(out, " ", warning)?; + } + } + for warning in &page.warnings { + write_warning(out, " ", warning)?; + } + } + writeln!(out, "ok: {}", report.ok).map_err(write_err) +} + +fn status_label(status: SlotStatus) -> &'static str { + match status { + SlotStatus::Confirmed => "confirmed", + SlotStatus::Partial => "partial", + SlotStatus::Missing => "missing", + } +} + +#[allow( + clippy::needless_pass_by_value, + reason = "used as a map_err fn that receives io::Error by value" +)] +fn write_err(error: io::Error) -> String { + format!("failed to write command output: {error}") +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + use crate::ad_templates::compare::{DomEvidence, GptSlotEvidence}; + use crate::commands::audit::collector::CollectedPage; + + struct FakeCollector { + pages: HashMap>, + } + + impl FakeCollector { + fn page(requested: &str, final_url: &str, evidence: BrowserAdEvidence) -> Self { + let mut pages = HashMap::new(); + pages.insert( + requested.to_string(), + Ok(CollectedPage { + final_url: url::Url::parse(final_url).expect("valid final url"), + title: String::new(), + script_count: 0, + resource_count: 0, + warnings: Vec::new(), + ad_evidence: Some(evidence), + }), + ); + Self { pages } + } + + fn with_error(mut self, requested: &str, message: &str) -> Self { + self.pages + .insert(requested.to_string(), Err(message.to_string())); + self + } + } + + impl AuditCollector for FakeCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.pages + .get(request.url.as_str()) + .cloned() + .unwrap_or_else(|| Err(format!("no fake page for {}", request.url))) + } + } + + fn news_config() -> CreativeOpportunitiesConfig { + let toml = "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + config + } + + fn confirmed_news_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: vec![DomEvidence { + dom_id: "ad-atf-0".to_string(), + phase: EvidencePhase::InitialLoad, + }], + gpt_slots: vec![GptSlotEvidence { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(300, 250)], + phase: EvidencePhase::InitialLoad, + }], + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn report_for( + collector: &dyn AuditCollector, + auction_enabled: bool, + strict: bool, + urls: &[&str], + ) -> VerificationReport { + report_for_with_options( + collector, + auction_enabled, + urls, + VerifyOptions { + strict, + scroll: false, + allow_cross_origin_redirect: false, + }, + ) + } + + fn report_for_with_options( + collector: &dyn AuditCollector, + auction_enabled: bool, + urls: &[&str], + options: VerifyOptions, + ) -> VerificationReport { + let config = news_config(); + let parsed: Vec = urls + .iter() + .map(|url| url::Url::parse(url).expect("valid url")) + .collect(); + build_report( + collector, + Some(&config), + auction_enabled, + &parsed, + options, + &[], + ) + } + + #[test] + fn verify_uses_final_url_for_matching_after_redirect() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, false, &["https://www.example.com/"]); + let json = serde_json::to_value(&report).expect("should serialize"); + + assert_eq!(json["pages"][0]["path"], "/news/story"); + assert_eq!(json["pages"][0]["slots"][0]["status"], "confirmed"); + let warnings = json["pages"][0]["warnings"] + .as_array() + .expect("warnings array"); + assert!( + warnings.iter().any(|w| w["code"] == "redirected"), + "redirect should emit a `redirected` warning" + ); + } + + #[test] + fn cross_origin_redirect_is_rejected_even_when_paths_match() { + // Same path on a different origin: the redirect warning would not fire, + // so without the origin check this unrelated page's evidence would + // satisfy --strict. + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://impostor.example.net/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!(!report.ok, "a cross-origin redirect must not report ok"); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][0]["error"]["code"], "cross_origin_redirect"); + assert!( + json["pages"][0]["slots"] + .as_array() + .expect("slots array") + .is_empty(), + "off-origin evidence must not be reported as slot verification" + ); + } + + #[test] + fn cross_origin_redirect_is_accepted_with_explicit_opt_in() { + let collector = FakeCollector::page( + "https://example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for_with_options( + &collector, + true, + &["https://example.com/news/story"], + VerifyOptions { + strict: true, + scroll: false, + allow_cross_origin_redirect: true, + }, + ); + + assert!( + report.ok, + "an opted-in apex -> www redirect should verify normally" + ); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn same_origin_path_redirect_still_verifies() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, true, &["https://www.example.com/"]); + + assert!( + report.ok, + "a same-origin redirect should still be verified, not refused" + ); + } + + #[test] + fn confirmed_page_is_ok_in_default_mode() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!(report.ok, "confirmed page should be ok"); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn strict_missing_slot_fails() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + !report.ok, + "strict mode with a missing slot should not be ok" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + // auction disabled -> runtime expected No -> strict does not fail on missing. + let report = report_for( + &collector, + false, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + report.ok, + "missing slot must not fail strict when auction is disabled" + ); + assert_eq!( + report.pages[0].runtime_ad_stack_expected, + Some(RuntimeAdStackExpectedJson::No) + ); + } + + #[test] + fn multi_url_page_error_sets_ok_false() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ) + .with_error("https://www.example.com/broken", "navigation failed"); + let report = report_for( + &collector, + true, + false, + &[ + "https://www.example.com/news/story", + "https://www.example.com/broken", + ], + ); + + assert!(!report.ok, "a page-level error sets ok=false"); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][1]["error"]["code"], "navigation_failed"); + assert!(json["pages"][1]["final_url"].is_null()); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs new file mode 100644 index 000000000..ab998adec --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -0,0 +1,610 @@ +//! Chrome/Chromium-backed implementation of [`AuditCollector`] using +//! `chromiumoxide` (CDP). +//! +//! The collector installs optional pre-navigation init scripts, sets any +//! operator-supplied cookies, navigates, waits for the page to settle, optionally +//! scrolls, and reads back a bounded set of evidence. It never *captures* page +//! HTML, cookies, or storage; supplied cookies are only *sent* to carry an +//! existing session past origin gates. + +use std::time::Duration; + +use chromiumoxide::browser::{Browser, BrowserConfig}; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; +use chromiumoxide::page::Page; +use futures::StreamExt as _; + +use crate::ad_templates::compare::BrowserAdEvidence; +use crate::ad_templates::output::Warning; +use crate::commands::audit::collector::{ + AuditCollector, BrowserCollectRequest, BrowserOpts, CollectedPage, +}; + +/// Candidate Chrome/Chromium executable names searched on `PATH`. +const CHROME_NAMES: &[&str] = &[ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + "chrome", +]; + +/// Poll interval while waiting for the page network to settle, in milliseconds. +const SETTLE_POLL_MS: u64 = 250; +/// Hard cap on page navigation so a stalled load cannot hang the audit. +const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); +/// Hard cap per decoded evidence list, mirroring the collector script's +/// `__ts_max_entries`, so a hostile page cannot inflate CLI memory. +const MAX_EVIDENCE_ENTRIES: usize = 1024; +/// Hard cap on browser teardown so a wedged Chrome cannot hang the audit. +const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); +/// Default quiet window (no new resources) marking the page settled. +const DEFAULT_SETTLE_QUIET_MS: u64 = 750; +/// Default hard cap on settling so slow/ad-heavy pages still terminate. +const DEFAULT_SETTLE_MAX_MS: u64 = 10_000; + +/// Page-settle timing thresholds. +#[derive(Debug, Clone, Copy)] +struct SettleConfig { + /// Quiet window with no new resources marking the page settled. + quiet: Duration, + /// Hard cap on total settle time. + max: Duration, +} + +/// A `chromiumoxide`-backed page collector launching a local Chrome/Chromium. +#[derive(Debug, Clone)] +pub struct BrowserCollector { + /// Explicit Chrome/Chromium executable override (else `$CHROME`, else auto-detect). + chrome: Option, + /// Quiet window marking the page settled. + settle_quiet: Duration, + /// Hard cap on settling. + settle_max: Duration, + /// Navigate to origins with invalid TLS certificates (dangerous opt-in). + accept_invalid_certs: bool, +} + +impl Default for BrowserCollector { + fn default() -> Self { + Self::new() + } +} + +impl BrowserCollector { + /// Creates a collector with default tuning and auto-detected Chrome. + #[must_use] + pub fn new() -> Self { + Self { + chrome: None, + settle_quiet: Duration::from_millis(DEFAULT_SETTLE_QUIET_MS), + settle_max: Duration::from_millis(DEFAULT_SETTLE_MAX_MS), + accept_invalid_certs: false, + } + } + + /// Creates a collector from operator-supplied browser options. + #[must_use] + pub fn from_opts(opts: &BrowserOpts) -> Self { + Self { + chrome: opts.chrome.clone(), + settle_quiet: Duration::from_millis(opts.settle_quiet_ms), + settle_max: Duration::from_millis(opts.settle_max_ms), + accept_invalid_certs: opts.danger_accept_invalid_certs, + } + } +} + +/// Resolves the Chrome/Chromium executable to launch. +/// +/// Precedence: explicit `--chrome` override, then the `CHROME` environment +/// variable, then auto-detection on `PATH` and standard install locations. +fn resolve_chrome(override_path: Option<&std::path::Path>) -> Result { + if let Some(path) = override_path { + return if path.is_file() { + Ok(path.to_path_buf()) + } else { + Err(format!( + "--chrome path does not point to a file: {}", + path.display() + )) + }; + } + if let Ok(env_path) = std::env::var("CHROME") { + let path = std::path::PathBuf::from(&env_path); + return if path.is_file() { + Ok(path) + } else { + Err(format!("CHROME={env_path} does not point to a file")) + }; + } + find_chrome() +} + +/// Auto-detects a Chrome/Chromium executable. +/// +/// Searches `PATH` by common names first, then well-known per-OS install +/// locations (e.g. the macOS `.app` bundle, which is not on `PATH`). +fn find_chrome() -> Result { + if let Some(path) = CHROME_NAMES.iter().find_map(|name| which::which(name).ok()) { + return Ok(path); + } + if let Some(path) = well_known_chrome_paths() + .into_iter() + .find(|path| path.is_file()) + { + return Ok(path); + } + Err(format!( + "could not find Chrome/Chromium on PATH or in standard install locations (looked for: {})", + CHROME_NAMES.join(", ") + )) +} + +/// Well-known absolute Chrome/Chromium install locations for the host OS. +fn well_known_chrome_paths() -> Vec { + let mut paths = Vec::new(); + + #[cfg(target_os = "macos")] + { + const APPS: &[&str] = &[ + "Google Chrome.app/Contents/MacOS/Google Chrome", + "Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "Chromium.app/Contents/MacOS/Chromium", + ]; + for app in APPS { + paths.push(std::path::PathBuf::from(format!("/Applications/{app}"))); + if let Ok(home) = std::env::var("HOME") { + paths.push(std::path::PathBuf::from(format!( + "{home}/Applications/{app}" + ))); + } + } + } + + #[cfg(target_os = "linux")] + { + for path in [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/snap/bin/chromium", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + #[cfg(target_os = "windows")] + { + for path in [ + r"C:\Program Files\Google\Chrome\Application\chrome.exe", + r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + paths +} + +impl AuditCollector for BrowserCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + // HTTP(S) scheme is enforced by the CLI value parser before we get here. + let chrome = resolve_chrome(self.chrome.as_deref())?; + let profile = tempfile::tempdir() + .map_err(|error| format!("failed to create browser profile dir: {error}"))?; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("failed to build browser runtime: {error}"))?; + + let settle = SettleConfig { + quiet: self.settle_quiet, + max: self.settle_max, + }; + + // chromiumoxide logs unparseable CDP events at WARN (its bundled CDP schema + // lags newer Chrome). These are benign; quiet them for the browser session + // so audit output stays clean, then restore the prior threshold. + let previous_level = log::max_level(); + log::set_max_level(log::LevelFilter::Error); + let accept_invalid_certs = self.accept_invalid_certs; + let result = runtime.block_on(async move { + collect( + &chrome, + profile.path(), + request, + settle, + accept_invalid_certs, + ) + .await + }); + log::set_max_level(previous_level); + result + } +} + +/// Drives a single page collection on the current-thread runtime. +async fn collect( + chrome: &std::path::Path, + profile_dir: &std::path::Path, + request: BrowserCollectRequest, + settle_config: SettleConfig, + accept_invalid_certs: bool, +) -> Result { + // chromiumoxide defaults to ignoring TLS errors. The audit sends + // operator-supplied session cookies and treats what it reads back as + // verification evidence, so a certificate-invalid impersonator could both + // harvest the session and fabricate the evidence. Validate certificates + // unless the operator explicitly opts out. + let mut builder = BrowserConfig::builder() + .chrome_executable(chrome) + .user_data_dir(profile_dir); + if !accept_invalid_certs { + builder = builder.respect_https_errors(); + } + let config = builder + .build() + .map_err(|error| format!("failed to build browser config: {error}"))?; + + let (mut browser, mut handler) = Browser::launch(config) + .await + .map_err(|error| format!("failed to launch browser: {error}"))?; + + // Drive the CDP event loop for the duration of the session. + let handler_task = tokio::spawn(async move { while handler.next().await.is_some() {} }); + + let result = collect_with_browser(&browser, request, settle_config).await; + + // Best-effort teardown; ignore errors since we already have a result, but + // bound it so a Chrome that ignores `close` cannot hang the command. + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.close()).await; + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.wait()).await; + handler_task.abort(); + + result +} + +async fn collect_with_browser( + browser: &Browser, + request: BrowserCollectRequest, + settle_config: SettleConfig, +) -> Result { + let mut warnings = Vec::new(); + + // Open a blank page first so init scripts are installed before the real + // document loads (evaluate-on-new-document applies to subsequent navigations). + let page = browser + .new_page("about:blank") + .await + .map_err(|error| format!("failed to open browser page: {error}"))?; + + for script in &request.init_scripts { + page.evaluate_on_new_document(script.clone()) + .await + .map_err(|error| format!("failed to install init script: {error}"))?; + } + + // Set operator-supplied cookies on the context before navigating so the + // origin sees an authenticated session on the first request. Scoping each to + // the request URL lets Chrome infer domain/path. + for (name, value) in &request.cookies { + let mut cookie = CookieParam::new(name.clone(), value.clone()); + cookie.url = Some(request.url.to_string()); + page.set_cookie(cookie) + .await + .map_err(|error| format!("failed to set cookie `{name}`: {error}"))?; + } + + tokio::time::timeout(NAVIGATION_TIMEOUT, page.goto(request.url.as_str())) + .await + .map_err(|_| format!("navigation to {} timed out", request.url))? + .map_err(|error| format!("failed to navigate to {}: {error}", request.url))?; + tokio::time::timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation()) + .await + .map_err(|_| format!("navigation to {} timed out", request.url))? + .map_err(|error| format!("failed to read main document navigation response: {error}"))?; + + settle(&page, settle_config).await; + + if request.scroll { + if request.collect_ad_evidence { + // Snapshot evidence before scrolling so entries already present at + // initial load keep phase "load"; the store dedups first-seen, so + // the post-scroll scrape only adds genuinely scroll-phase entries. + let _ = page + .evaluate( + "(typeof window.__tsCollectAdTemplateEvidence === 'function' \ + && window.__tsCollectAdTemplateEvidence(), null)", + ) + .await; + } + scroll_page(&page).await; + settle(&page, settle_config).await; + } + + let final_url = match page.url().await { + Ok(Some(url)) => url::Url::parse(&url).unwrap_or_else(|_| request.url.clone()), + _ => request.url.clone(), + }; + let title = page.get_title().await.ok().flatten().unwrap_or_default(); + let script_count = eval_usize(&page, "document.querySelectorAll('script').length").await; + let resource_count = resource_count(&page).await; + + let ad_evidence = if request.collect_ad_evidence { + extract_ad_evidence(&page, &mut warnings).await + } else { + None + }; + + Ok(CollectedPage { + final_url, + title, + script_count, + resource_count, + warnings, + ad_evidence, + }) +} + +/// Waits for the page network to go quiet after navigation or scroll. +/// +/// Polls the resource-entry count and returns once it stays unchanged for a +/// quiet window, or when the hard cap elapses — so ad-heavy pages finish loading +/// before evidence is read, without hanging on pages that never go idle. +async fn settle(page: &Page, config: SettleConfig) { + let start = std::time::Instant::now(); + let quiet_target = config.quiet; + let max = config.max; + let mut last = resource_count(page).await; + let mut quiet = Duration::ZERO; + while start.elapsed() < max { + tokio::time::sleep(Duration::from_millis(SETTLE_POLL_MS)).await; + let current = resource_count(page).await; + if current == last { + quiet += Duration::from_millis(SETTLE_POLL_MS); + if quiet >= quiet_target { + break; + } + } else { + quiet = Duration::ZERO; + last = current; + } + } +} + +/// Reads the number of resource timing entries observed so far. +async fn resource_count(page: &Page) -> usize { + eval_usize(page, "performance.getEntriesByType('resource').length").await +} + +/// Performs a deterministic stepped scroll to trigger lazy ad loading. +async fn scroll_page(page: &Page) { + // Mark subsequent observations as scroll-phase for the collector. + let _ = page.evaluate("window.__tsScrollPhase = true").await; + for fraction in ["0.33", "0.66", "1"] { + let script = format!( + "window.scrollTo(0, Math.floor(Math.max(document.body.scrollHeight, \ + document.documentElement.scrollHeight) * {fraction}))" + ); + let _ = page.evaluate(script).await; + tokio::time::sleep(Duration::from_millis(250)).await; + } + let _ = page.evaluate("window.scrollTo(0, 0)").await; +} + +/// Evaluates a numeric expression, returning 0 on any failure. +async fn eval_usize(page: &Page, expression: &str) -> usize { + page.evaluate(expression) + .await + .ok() + .and_then(|result| result.into_value::().ok()) + .unwrap_or(0) +} + +/// Reads and decodes `window.__tsAdTemplateEvidence`, warning (not failing) on a +/// decode error. +async fn extract_ad_evidence( + page: &Page, + warnings: &mut Vec, +) -> Option { + // Trigger the on-demand DOM + getSlots scrape, then read the evidence object. + let value = page + .evaluate( + "(typeof window.__tsCollectAdTemplateEvidence === 'function' \ + ? window.__tsCollectAdTemplateEvidence() \ + : (window.__tsAdTemplateEvidence || null))", + ) + .await + .ok() + .and_then(|result| result.into_value::().ok()); + + match value { + Some(serde_json::Value::Null) | None => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + Some(value) => match serde_json::from_value::(value) { + Ok(mut evidence) => { + // Defense in depth: the injected script caps these lists, but the + // page owns that store, so re-cap after decode. + evidence.dom_ids.truncate(MAX_EVIDENCE_ENTRIES); + evidence.gpt_slots.truncate(MAX_EVIDENCE_ENTRIES); + evidence.aps_calls.truncate(MAX_EVIDENCE_ENTRIES); + evidence.warnings.truncate(MAX_EVIDENCE_ENTRIES); + Some(evidence) + } + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence: {error}"), + }); + None + } + }, + } +} + +#[cfg(test)] +mod tests { + use std::io::Write as _; + + use super::*; + use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, build_ad_template_init_script, + }; + + /// Whether a local Chrome/Chromium is available to run browser fixture tests. + fn chrome_available() -> bool { + find_chrome().is_ok() + } + + #[test] + fn well_known_chrome_paths_are_known_for_this_os() { + // macOS/Linux/Windows each have candidate paths; guards the cfg branches. + assert!( + !well_known_chrome_paths().is_empty(), + "supported OSes should list candidate Chrome install paths" + ); + } + + /// A self-contained page that stubs just enough of GPT (no network) for the + /// collector to observe a defined slot via the wrapped `defineSlot` and the + /// `getSlots()` scrape. + const GPT_FIXTURE: &str = r#" + + + +
+ + + +"#; + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn collects_gpt_slot_from_local_fixture() { + if !chrome_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + aps_slot_ids: Vec::new(), + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: false, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence + .gpt_slots + .iter() + .any(|slot| slot.gam_unit_path == "/123/news/atf"), + "should capture the defined GPT slot" + ); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0"), + "should capture the configured-prefix DOM id" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn scroll_pass_keeps_initial_load_phase_for_load_time_evidence() { + if !chrome_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + aps_slot_ids: Vec::new(), + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: true, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + // The slot and DOM id exist at load time, so the pre-scroll snapshot + // must record them as initial-load even though a scroll pass ran. + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0" + && dom.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad), + "load-time DOM id should keep phase initial_load under --scroll" + ); + assert!( + evidence.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/news/atf" + && slot.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad + }), + "load-time GPT slot should keep phase initial_load under --scroll" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/browser_collector.rs deleted file mode 100644 index 87a2ccc2c..000000000 --- a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use chromiumoxide::ArcHttpRequest; -use chromiumoxide::browser::{Browser, BrowserConfig}; -use futures::StreamExt as _; -use serde::Deserialize; -use tempfile::TempDir; -use tokio::runtime::Builder; -use tokio::time::{sleep, timeout}; -use url::Url; -use which::which; - -use crate::commands::audit::collector::{ - AuditCollector, CollectedPage, CollectedRequest, CollectedScriptTag, -}; -use crate::error::{CliResult, report_error}; - -const SETTLE_QUIET_PERIOD: Duration = Duration::from_millis(750); -const SETTLE_POLL_INTERVAL: Duration = Duration::from_millis(250); -const SETTLE_MAX_WAIT: Duration = Duration::from_secs(6); -const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); -const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); -const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; -const RESOURCE_TIMING_BUFFER_WARNING: &str = - "browser resource timing buffer reached its default size; some network assets may be missing"; - -#[derive(Default)] -pub(crate) struct BrowserAuditCollector; - -impl AuditCollector for BrowserAuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult { - let runtime = Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Tokio runtime for browser audit: {error}" - )) - })?; - - runtime.block_on(collect_page_via_browser_async(target_url)) - } -} - -async fn collect_page_via_browser_async(target_url: &Url) -> CliResult { - let chrome_executable = find_browser_executable()?; - let user_data_dir = TempDir::new().map_err(|error| { - report_error(format!( - "failed to create temporary browser profile for audit: {error}" - )) - })?; - let config = BrowserConfig::builder() - .chrome_executable(chrome_executable) - .user_data_dir(user_data_dir.path()) - .new_headless_mode() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Chromium configuration for audit: {error}" - )) - })?; - - let (mut browser, mut handler) = Browser::launch(config).await.map_err(|error| { - report_error(format!( - "failed to launch Chrome/Chromium for audit: {error}" - )) - })?; - - let handler_task = tokio::spawn(async move { - while let Some(event) = handler.next().await { - if event.is_err() { - break; - } - } - }); - - let result = collect_page_from_browser(&mut browser, target_url).await; - - let close_result = timeout(BROWSER_CLOSE_TIMEOUT, browser.close()) - .await - .map_err(|_| report_error("timed out closing browser after audit")) - .and_then(|result| { - result.map_err(|error| { - report_error(format!("failed to close browser after audit: {error}")) - }) - }); - if close_result.is_err() { - handler_task.abort(); - } - let _ = handler_task.await; - - match (result, close_result) { - (Ok(collected), Ok(_)) => Ok(collected), - (Ok(_), Err(error)) | (Err(error), _) => Err(error), - } -} - -async fn collect_page_from_browser( - browser: &mut Browser, - target_url: &Url, -) -> CliResult { - let page = browser.new_page("about:blank").await.map_err(|error| { - report_error(format!("failed to create browser page for audit: {error}")) - })?; - - timeout(NAVIGATION_TIMEOUT, page.goto(target_url.as_str())) - .await - .map_err(|_| report_error(format!("timed out navigating to `{target_url}`")))? - .map_err(|error| report_error(format!("failed to navigate to `{target_url}`: {error}")))?; - - let navigation_response = timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation_response()) - .await - .map_err(|_| { - report_error(format!( - "timed out waiting for main document navigation response from `{target_url}`" - )) - })? - .map_err(|error| { - report_error(format!( - "failed to read main document navigation response: {error}" - )) - })?; - - let mut warnings = Vec::new(); - if let Some(warning) = validate_navigation_response(navigation_response)? { - warnings.push(warning); - } - if !wait_for_page_settle(&page).await? { - warnings.push( - "browser audit timed out while waiting for the page to settle; results may be partial" - .to_string(), - ); - } - - let final_url = page - .url() - .await - .map_err(|error| report_error(format!("failed to read final page URL: {error}")))? - .ok_or_else(|| report_error("browser page URL was empty after navigation"))?; - let page_title = page - .get_title() - .await - .map_err(|error| report_error(format!("failed to read page title: {error}")))?; - let html = page - .content() - .await - .map_err(|error| report_error(format!("failed to read rendered page HTML: {error}")))?; - - let script_tags: Vec = page - .evaluate( - r#"() => Array.from(document.scripts).map((script) => ({ - src: script.src || null, - inline_text: script.src ? null : (script.textContent || null), - }))"#, - ) - .await - .map_err(|error| report_error(format!("failed to read rendered script tags: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode rendered script tag data: {error}" - )) - })?; - - let network_requests: Vec = page - .evaluate( - r#"() => performance.getEntriesByType('resource').map((entry) => ({ - url: entry.name, - initiator_type: entry.initiatorType || null, - }))"#, - ) - .await - .map_err(|error| { - report_error(format!( - "failed to read browser performance resource entries: {error}" - )) - })? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode browser performance resource data: {error}" - )) - })?; - - if let Some(warning) = resource_timing_buffer_warning(network_requests.len()) { - warnings.push(warning.to_string()); - } - - Ok(CollectedPage { - requested_url: target_url.to_string(), - final_url, - page_title: page_title.filter(|title| !title.trim().is_empty()), - html, - script_tags: script_tags - .into_iter() - .map(|script| CollectedScriptTag { - src: script.src, - inline_text: script.inline_text.filter(|text| !text.trim().is_empty()), - }) - .collect(), - network_requests: network_requests - .into_iter() - .map(|entry| CollectedRequest { - url: entry.url, - resource_type: entry.initiator_type, - }) - .collect(), - warnings, - }) -} - -async fn wait_for_page_settle(page: &chromiumoxide::Page) -> CliResult { - let mut elapsed = Duration::ZERO; - let mut previous_count = None; - let mut stable_for = Duration::ZERO; - - while elapsed < SETTLE_MAX_WAIT { - let ready_state: String = page - .evaluate("document.readyState") - .await - .map_err(|error| report_error(format!("failed to read document ready state: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!("failed to decode document ready state: {error}")) - })?; - let resource_count: usize = page - .evaluate("performance.getEntriesByType('resource').length") - .await - .map_err(|error| report_error(format!("failed to read resource count: {error}")))? - .into_value() - .map_err(|error| report_error(format!("failed to decode resource count: {error}")))?; - - if ready_state == "complete" { - if previous_count == Some(resource_count) { - stable_for += SETTLE_POLL_INTERVAL; - } else { - stable_for = Duration::ZERO; - } - - if stable_for >= SETTLE_QUIET_PERIOD { - return Ok(true); - } - } - - previous_count = Some(resource_count); - sleep(SETTLE_POLL_INTERVAL).await; - elapsed += SETTLE_POLL_INTERVAL; - } - - Ok(false) -} - -fn validate_navigation_response(navigation_response: ArcHttpRequest) -> CliResult> { - let request = navigation_response - .ok_or_else(|| report_error("browser audit did not capture the main document response"))?; - - if let Some(failure_text) = &request.failure_text { - return Err(report_error(format!( - "main document request failed: {failure_text}" - ))); - } - - let response = request.response.as_ref().ok_or_else(|| { - report_error("browser audit did not capture the main document HTTP response") - })?; - - if is_successful_navigation_status(response.status) { - return Ok(None); - } - - Ok(Some(format!( - "audit request returned HTTP {} {} for `{}`; results may be partial", - response.status, response.status_text, response.url - ))) -} - -fn is_successful_navigation_status(status: i64) -> bool { - (200..400).contains(&status) -} - -fn resource_timing_buffer_warning(resource_count: usize) -> Option<&'static str> { - (resource_count >= RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD) - .then_some(RESOURCE_TIMING_BUFFER_WARNING) -} - -fn find_browser_executable() -> CliResult { - for candidate in browser_executable_path_candidates() { - if let Ok(path) = which(candidate) { - return Ok(path); - } - } - - for candidate in browser_executable_fallbacks() { - let candidate_path = Path::new(candidate); - if candidate_path.is_file() { - return Ok(candidate_path.to_path_buf()); - } - } - - Err(report_error( - "Chrome/Chromium was not found on PATH or in the standard local install locations checked by `ts audit`. Install a local Chrome or Chromium binary before running `ts audit`.", - )) -} - -fn browser_executable_path_candidates() -> &'static [&'static str] { - &[ - "google-chrome", - "google-chrome-stable", - "chromium", - "chromium-browser", - "chrome", - "Google Chrome", - "Google Chrome for Testing", - ] -} - -fn browser_executable_fallbacks() -> &'static [&'static str] { - #[cfg(target_os = "macos")] - { - &[ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/Applications/Chromium.app/Contents/MacOS/Chromium", - "/Applications/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", - ] - } - - #[cfg(target_os = "linux")] - { - &[ - "/usr/bin/google-chrome", - "/usr/bin/google-chrome-stable", - "/usr/bin/chromium", - "/usr/bin/chromium-browser", - "/snap/bin/chromium", - ] - } - - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - { - &[] - } -} - -#[derive(Debug, Deserialize)] -struct BrowserScriptTag { - src: Option, - inline_text: Option, -} - -#[derive(Debug, Deserialize)] -struct BrowserPerformanceEntry { - url: String, - initiator_type: Option, -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use chromiumoxide::cdp::browser_protocol::network::{Headers, RequestId, Response}; - use chromiumoxide::cdp::browser_protocol::security::SecurityState; - use chromiumoxide::handler::http::HttpRequest; - - use super::*; - - #[test] - fn successful_navigation_status_allows_redirects_but_rejects_errors() { - assert!(is_successful_navigation_status(200)); - assert!(is_successful_navigation_status(302)); - assert!(is_successful_navigation_status(399)); - assert!(!is_successful_navigation_status(199)); - assert!(!is_successful_navigation_status(400)); - assert!(!is_successful_navigation_status(500)); - } - - #[test] - fn navigation_response_returns_warning_for_http_error_status() { - let warning = - validate_navigation_response(navigation_response_with_status(403, "Forbidden")) - .expect("should validate navigation response") - .expect("should return warning for HTTP error status"); - - assert_eq!( - warning, - "audit request returned HTTP 403 Forbidden for `https://example.com/`; results may be partial", - "should warn and continue when the main document returns an HTTP error" - ); - } - - #[test] - fn resource_timing_buffer_warning_starts_at_threshold() { - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD - 1), - None, - "should not warn before the resource timing buffer threshold" - ); - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD), - Some(RESOURCE_TIMING_BUFFER_WARNING), - "should warn when the resource timing buffer reaches the threshold" - ); - } - - #[test] - fn browser_path_candidates_include_common_names() { - let candidates = browser_executable_path_candidates(); - - assert!(candidates.contains(&"google-chrome")); - assert!(candidates.contains(&"chromium")); - assert!(candidates.contains(&"Google Chrome for Testing")); - } - - fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { - let mut request = - HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); - request.response = Some( - Response::builder() - .url("https://example.com/") - .status(status) - .status_text(status_text) - .headers(Headers::default()) - .mime_type("text/html") - .charset("utf-8") - .connection_reused(false) - .connection_id(1.0) - .encoded_data_length(0.0) - .security_state(SecurityState::Secure) - .build() - .expect("should build navigation response"), - ); - - Some(Arc::new(request)) - } -} diff --git a/crates/trusted-server-cli/src/commands/audit/collector.rs b/crates/trusted-server-cli/src/commands/audit/collector.rs index 314ae54fc..ff7a45867 100644 --- a/crates/trusted-server-cli/src/commands/audit/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/collector.rs @@ -1,41 +1,199 @@ -use serde::{Deserialize, Serialize}; -use url::Url; +//! Collector abstraction shared by the generic page audit and the ad-template +//! verifier. +//! +//! Decoupling collection behind [`AuditCollector`] lets the verifier orchestration +//! (Task 9) be tested with an in-memory fake collector, with no Chrome dependency. -use crate::error::CliResult; +use std::path::PathBuf; -pub(crate) trait AuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult; +use clap::Args; + +use crate::ad_templates::compare::BrowserAdEvidence; + +/// Operator-tunable browser options shared by `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// These are audit-tool knobs, not publisher runtime config, so they live on the +/// CLI (flags / `CHROME` env) rather than in `trusted-server.toml`. +#[derive(Debug, Clone, Args)] +pub struct BrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then + /// auto-detection on `PATH` and standard install locations. + #[arg(long)] + pub chrome: Option, + /// Quiet window in milliseconds (no new network resources) that marks the + /// page settled. + #[arg(long, default_value_t = 750)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = 10_000)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as verification evidence, so an invalid + /// certificate could mean an impersonator is harvesting the session and + /// fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// A request to collect a single page. +#[derive(Debug, Clone)] +pub struct BrowserCollectRequest { + /// The URL to navigate to. + pub url: url::Url, + /// Pre-navigation init scripts (evaluate-on-new-document). Empty for a plain + /// page audit; the ad-template verifier supplies the read-only collector here. + pub init_scripts: Vec, + /// Whether to perform the deterministic scroll pass after settle. + pub scroll: bool, + /// Whether to extract `window.__tsAdTemplateEvidence` after settle/scroll. + pub collect_ad_evidence: bool, + /// Operator-supplied `(name, value)` cookies set on the browser context + /// before navigation, scoped to the request URL. Used to carry an existing + /// authenticated session (e.g. a valid bot-protection clearance cookie) so + /// the origin serves the real page instead of a challenge. The collector + /// only sends these; it never reads cookies back. + pub cookies: Vec<(String, String)>, +} + +/// The result of collecting a single page. +#[derive(Debug, Clone)] +pub struct CollectedPage { + /// The final URL after redirects. + pub final_url: url::Url, + /// The page title. + pub title: String, + /// Number of `"#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1712,6 +1876,7 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1750,6 +1915,7 @@ mod tests { let request_host = "proxy.test-publisher.example.com"; let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.test-publisher.example.com".to_string(), request_host: request_host.to_string(), request_scheme: "https".to_string(), @@ -1802,6 +1968,7 @@ mod tests { // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1832,6 +1999,7 @@ mod tests { // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1853,6 +2021,41 @@ mod tests { ); } + #[test] + fn bodyless_marker_mode_emits_an_owned_terminal_seam_even_after_source_bytes() { + const MARKER: &str = ""; + let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::Marker(MARKER.to_string()), + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), + max_buffered_body_bytes: 16 * 1024 * 1024, + gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, + }; + let source = + format!(r#""#); + + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(source.as_bytes(), true) + .expect("should process bodyless HTML"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + + assert_eq!( + html.matches(MARKER).count(), + 2, + "one source occurrence plus one transform-owned seam must reach normalization" + ); + assert!( + html.ends_with(MARKER), + "the transform-owned fallback must be unambiguously terminal" + ); + } + #[test] fn response_size_does_not_grow_disproportionately() { // Processing must not expand HTML by more than 1.1× (accounts for the diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs index 5ad7011fe..1b0693e56 100644 --- a/crates/trusted-server-core/src/http_util.rs +++ b/crates/trusted-server-core/src/http_util.rs @@ -4,8 +4,10 @@ use edgezero_core::body::Body as EdgeBody; use error_stack::Report; use http::{Request, Response, StatusCode, header}; use sha2::{Digest as _, Sha256}; +use std::time::Duration; use subtle::ConstantTimeEq as _; +use crate::cache_policy::{CachePolicy, EdgeCacheHeader}; use crate::constants::INTERNAL_HEADERS; use crate::error::TrustedServerError; use crate::platform::ClientInfo; @@ -274,43 +276,41 @@ pub fn serve_static_with_etag( body: &str, req: &Request, content_type: &str, + edge_header: EdgeCacheHeader, ) -> Response { - // Compute ETag for conditional caching let hash = Sha256::digest(body.as_bytes()); let etag = format!("\"sha256-{}\"", hex::encode(hash)); + let short_policy = CachePolicy::public_short_with_stale( + Duration::from_secs(300), + Duration::from_secs(60), + Duration::from_secs(86_400), + ); - // If-None-Match handling for 304 responses if let Some(if_none_match) = req .headers() .get(header::IF_NONE_MATCH) .and_then(|h| h.to_str().ok()) && if_none_match == etag { - return Response::builder() - .status(StatusCode::NOT_MODIFIED) - .header(header::ETAG, &etag) - .header( - header::CACHE_CONTROL, - "public, max-age=300, s-maxage=300, stale-while-revalidate=60, stale-if-error=86400", - ) - .header("surrogate-control", "max-age=300") - .header(header::VARY, "Accept-Encoding") - .body(EdgeBody::empty()) - .expect("should build 304 static response"); - } - - Response::builder() + let mut response = Response::builder() + .status(StatusCode::NOT_MODIFIED) + .header(header::ETAG, &etag) + .header(header::VARY, "Accept-Encoding") + .body(EdgeBody::empty()) + .expect("should build 304 static response"); + short_policy.apply_to_headers(response.headers_mut(), edge_header); + return response; + } + + let mut response = Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, content_type) - .header( - header::CACHE_CONTROL, - "public, max-age=300, s-maxage=300, stale-while-revalidate=60, stale-if-error=86400", - ) - .header("surrogate-control", "max-age=300") .header(header::ETAG, &etag) .header(header::VARY, "Accept-Encoding") .body(EdgeBody::from(body.as_bytes())) - .expect("should build static response") + .expect("should build static response"); + short_policy.apply_to_headers(response.headers_mut(), edge_header); + response } /// Encrypts a URL using XChaCha20-Poly1305 with a key derived from the publisher `proxy_secret`. diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 4fed9278d..fba758973 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -23,8 +23,8 @@ use crate::auction::types::{ }; use crate::error::TrustedServerError; use crate::integrations::{ - IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, - UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, + IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, + IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, ensure_integration_backend_with_timeout, predict_integration_backend_name, }; use crate::openrtb::{ @@ -114,6 +114,17 @@ addEventListener('message',receive); "#; +/// Rendering owner for selected APS bids. +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ApsRenderingMode { + /// Render through Trusted Server's opaque static renderer route. + #[default] + TrustedServer, + /// Delegate rendering to the publisher's explicit browser hook. + PublisherNative, +} + /// Configuration for the APS `OpenRTB` integration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] #[validate(schema(function = "validate_inventory_identity_override"))] @@ -141,6 +152,9 @@ pub struct ApsConfig { /// Whether APS script creatives are eligible before winner selection. #[serde(default)] pub allow_script_creatives: bool, + /// Rendering owner for selected APS bids. + #[serde(default)] + pub rendering_mode: ApsRenderingMode, /// APS-authorized inventory domain used instead of the deployment hostname. #[serde(default, skip_serializing_if = "Option::is_none")] #[validate(custom(function = "validate_inventory_domain"))] @@ -314,6 +328,7 @@ impl Default for ApsConfig { timeout_ms: default_timeout_ms(), debug: false, allow_script_creatives: false, + rendering_mode: ApsRenderingMode::TrustedServer, inventory_domain: None, inventory_page_origin: None, } @@ -1184,7 +1199,9 @@ impl AuctionProvider for ApsAuctionProvider { } #[derive(Debug)] -struct ApsRendererIntegration; +struct ApsRendererIntegration { + rendering_mode: ApsRenderingMode, +} #[async_trait(?Send)] impl IntegrationProxy for ApsRendererIntegration { @@ -1193,7 +1210,10 @@ impl IntegrationProxy for ApsRendererIntegration { } fn routes(&self) -> Vec { - vec![IntegrationEndpoint::get(APS_RENDERER_ROUTE)] + (self.rendering_mode == ApsRenderingMode::TrustedServer) + .then(|| IntegrationEndpoint::get(APS_RENDERER_ROUTE)) + .into_iter() + .collect() } async fn handle( @@ -1225,6 +1245,22 @@ impl IntegrationProxy for ApsRendererIntegration { } } +impl IntegrationHeadInjector for ApsRendererIntegration { + fn integration_id(&self) -> &'static str { + APS_INTEGRATION_ID + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + (self.rendering_mode == ApsRenderingMode::PublisherNative) + .then(|| { + "" + .to_string() + }) + .into_iter() + .collect() + } +} + /// Register the APS static renderer endpoint when APS is enabled. /// /// # Errors @@ -1233,16 +1269,21 @@ impl IntegrationProxy for ApsRendererIntegration { pub fn register( settings: &Settings, ) -> Result, Report> { - let Some(_config) = settings.integration_config::(APS_INTEGRATION_ID)? else { + let Some(config) = settings.integration_config::(APS_INTEGRATION_ID)? else { return Ok(None); }; - let integration = Arc::new(ApsRendererIntegration); - Ok(Some( - IntegrationRegistration::builder(APS_INTEGRATION_ID) - .with_proxy(integration) - .without_js() - .build(), - )) + let integration = Arc::new(ApsRendererIntegration { + rendering_mode: config.rendering_mode, + }); + let registration = IntegrationRegistration::builder(APS_INTEGRATION_ID) + .without_js() + .with_head_injector(integration.clone()); + let registration = if config.rendering_mode == ApsRenderingMode::TrustedServer { + registration.with_proxy(integration) + } else { + registration + }; + Ok(Some(registration.build())) } /// Register the APS auction provider when enabled. @@ -1273,6 +1314,7 @@ mod tests { UserInfo, }; use crate::consent::ConsentContext; + use crate::integrations::IntegrationDocumentState; use crate::openrtb::{Eid, Uid}; use crate::platform::GeoInfo; use crate::platform::test_support::{ @@ -1289,6 +1331,7 @@ mod tests { timeout_ms: 800, debug: false, allow_script_creatives: false, + rendering_mode: ApsRenderingMode::TrustedServer, inventory_domain: None, inventory_page_origin: None, } @@ -1405,6 +1448,7 @@ mod tests { assert!(!canonical.debug); assert!(debug.debug); assert!(!canonical.allow_script_creatives); + assert_eq!(canonical.rendering_mode, ApsRenderingMode::TrustedServer); assert!(canonical.endpoint.ends_with("/e/pb/bid")); } @@ -1466,6 +1510,14 @@ mod tests { })) .is_err() ); + assert!( + serde_json::from_value::(json!({ + "account_id": "example-account", + "rendering_mode": "unsupported" + })) + .is_err(), + "should reject an unknown APS rendering mode" + ); for endpoint in [ "http://aps.example/e/pb/bid", "https://", @@ -2319,7 +2371,9 @@ mod tests { #[test] fn registers_and_serves_only_static_renderer_route() { - let integration = ApsRendererIntegration; + let integration = ApsRendererIntegration { + rendering_mode: ApsRenderingMode::TrustedServer, + }; let routes = integration.routes(); assert_eq!(routes.len(), 1, "should register one route"); assert_eq!(routes[0].method, Method::GET); @@ -2374,9 +2428,55 @@ mod tests { assert_eq!(registration.integration_id, APS_INTEGRATION_ID); assert_eq!(registration.proxies.len(), 1); + assert_eq!(registration.head_injectors.len(), 1); assert!(registration.js_disabled); } + #[test] + fn publisher_native_config_registers_hook_mode_without_renderer_route() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + APS_INTEGRATION_ID, + &json!({ + "enabled": true, + "account_id": "example-account", + "rendering_mode": "publisher_native" + }), + ) + .expect("should insert native APS config"); + + let registration = register(&settings) + .expect("should register APS") + .expect("should return enabled registration"); + assert!( + registration.proxies.is_empty(), + "should not register the static renderer" + ); + assert_eq!(registration.head_injectors.len(), 1); + + let integration = ApsRendererIntegration { + rendering_mode: ApsRenderingMode::PublisherNative, + }; + assert!( + integration.routes().is_empty(), + "should expose no renderer route" + ); + let document_state = IntegrationDocumentState::default(); + let context = IntegrationHtmlContext { + request_host: "publisher.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + assert_eq!( + integration.head_inserts(&context), + vec![""], + "should inject only the native-mode marker" + ); + } + #[test] fn config_without_enabled_does_not_register_provider_or_renderer() { let mut settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 3e8f021fe..2c2c94ba7 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -68,6 +68,10 @@ pub struct GptConfig { #[serde(default = "default_enabled")] pub enabled: bool, + /// Enable page-level `ts=true` delivery attribution in GAM. + #[serde(default)] + pub gam_attribution_enabled: bool, + /// URL for the GPT bootstrap script (default: Google's CDN). #[serde(default = "default_script_url")] #[validate(url)] @@ -487,10 +491,17 @@ impl IntegrationHeadInjector for GptIntegration { /// route changes (see `auction/endpoints.rs`). /// The `POST /auction` endpoint is not involved in scroll or refresh flows. fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + let gam_attribution_flag = if self.config.gam_attribution_enabled { + "window.__tsjs_gam_attribution_enabled=true;" + } else { + "" + }; + let mut scripts = vec![ - "" - .to_string(), + format!( + "" + ), format!("", GPT_BOOTSTRAP_JS), ]; @@ -508,6 +519,14 @@ impl IntegrationHeadInjector for GptIntegration { scripts } + + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + if self.config.gam_attribution_enabled { + vec![("data-ts-gam-attribution", "true")] + } else { + Vec::new() + } + } } /// Inline `window.tsjs.adInit` bootstrap injected at `` so the bids @@ -549,6 +568,7 @@ mod tests { fn test_config() -> GptConfig { GptConfig { enabled: true, + gam_attribution_enabled: false, script_url: default_script_url(), cache_ttl_seconds: 3600, rewrite_script: true, @@ -573,6 +593,29 @@ mod tests { .expect("should build HTTP request") } + #[test] + fn gam_attribution_defaults_to_disabled() { + let config: GptConfig = + serde_json::from_value(serde_json::json!({})).expect("should parse defaults"); + + assert!(!config.gam_attribution_enabled); + } + + #[test] + fn gam_attribution_deserializes_explicit_values() { + let disabled: GptConfig = serde_json::from_value(serde_json::json!({ + "gam_attribution_enabled": false + })) + .expect("should parse explicit false"); + let enabled: GptConfig = serde_json::from_value(serde_json::json!({ + "gam_attribution_enabled": true + })) + .expect("should parse explicit true"); + + assert!(!disabled.gam_attribution_enabled); + assert!(enabled.gam_attribution_enabled); + } + // -- URL detection -- #[test] @@ -1146,6 +1189,38 @@ mod tests { "", "should set the enable flag and call the GPT shim activation function" ); + assert!( + integration.tsjs_script_tag_attributes().is_empty(), + "should not authorize GAM attribution metadata by default" + ); + } + + #[test] + fn gam_attribution_true_adds_both_activation_signals_without_a_new_insert() { + let integration = GptIntegration::new(GptConfig { + gam_attribution_enabled: true, + ..test_config() + }); + let document_state = IntegrationDocumentState::default(); + let context = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + document_state: &document_state, + }; + + let inserts = integration.head_inserts(&context); + + assert_eq!(inserts.len(), 2, "should not add another head insert"); + assert!( + inserts[0].contains("window.__tsjs_gam_attribution_enabled=true;"), + "should activate the early bootstrap marker" + ); + assert_eq!( + integration.tsjs_script_tag_attributes(), + vec![("data-ts-gam-attribution", "true")], + "should authorize the bundle fallback on the publisher tag" + ); } #[test] @@ -1352,6 +1427,35 @@ mod tests { ); } + #[test] + fn head_inserts_queue_gam_attribution_before_guard_and_ad_requests() { + let targeting_index = GPT_BOOTSTRAP_JS + .find("gpt.setConfig({ targeting: { ts: 'true' } })") + .expect("should apply the fixed page-level GAM targeting pair"); + let guard_index = GPT_BOOTSTRAP_JS + .find("if (ts.adInit) return;") + .expect("should retain the preinstalled adInit guard"); + let display_index = GPT_BOOTSTRAP_JS + .find("googletag.display(divId);") + .expect("should retain the executable GPT display call"); + let refresh_index = GPT_BOOTSTRAP_JS + .find("googletag.pubads().refresh(slotsNeedingRefresh);") + .expect("should retain the bounded GPT refresh call"); + + assert!( + targeting_index < guard_index, + "should enqueue attribution before the preinstalled adInit guard" + ); + assert!( + targeting_index < display_index, + "should enqueue attribution before the executable display call" + ); + assert!( + targeting_index < refresh_index, + "should enqueue attribution before the executable refresh call" + ); + } + #[test] fn head_injector_integration_id() { let integration = GptIntegration::new(test_config()); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 86b51ffa7..909686ceb 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -17,6 +17,24 @@ (function () { if (typeof window === "undefined") return; var ts = (window.tsjs = window.tsjs || {}); + var tag; + + if (window.__tsjs_gam_attribution_enabled === true) { + tag = window.googletag = window.googletag || { cmd: [] }; + tag.cmd = tag.cmd || []; + tag.cmd.push(function () { + try { + var gpt = window.googletag; + if (gpt && typeof gpt.setConfig === "function") { + // "ts" is the fixed GAM key, not the local window.tsjs alias. + gpt.setConfig({ targeting: { ts: 'true' } }); + } + } catch (_) { + // Attribution must not interrupt the existing bootstrap queue. + } + }); + } + if (ts.adInit) return; // Track whether the publisher disabled GPT initial load. Read the effective @@ -38,7 +56,9 @@ return true; } - (window.googletag = window.googletag || { cmd: [] }).cmd.push(function () { + tag = tag || (window.googletag = window.googletag || { cmd: [] }); + tag.cmd = tag.cmd || []; + tag.cmd.push(function () { var gpt = window.googletag; syncInitialLoadDisabled(gpt); if ( @@ -94,8 +114,12 @@ // and deliberately identical to the bundle scheduler — the impression is // spent on a viewed tab, and the post-hydration guarantee holds whenever // the request is actually issued. - ts.scheduleInitialAdInit = function (initialBids) { + ts.scheduleInitialAdInit = function (initialBids, initialSlots) { if ((ts.navGeneration || 0) !== 0) return; + // Slots are generation-guarded for the same reason the bids are: the + // shared-template seam sends both, and an assignment made before this call + // would overwrite a committed SPA navigation's slots. + if (initialSlots) ts.adSlots = initialSlots; if (initialBids) ts.bids = initialBids; var fire = function () { if ((ts.navGeneration || 0) !== 0) return; diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 5bd86d19e..5a8c5b680 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -12,9 +12,9 @@ use validator::Validate; use edgezero_core::body::Body as EdgeBody; +use crate::cache_policy::EDGE_CACHE_HEADER_NAMES; use crate::error::TrustedServerError; use crate::http_util::is_navigation_request; -use crate::response_privacy::CDN_CACHE_HEADERS; use crate::settings::{IntegrationConfig, Settings}; use crate::tsjs; @@ -114,6 +114,83 @@ impl GptDiagnosticsRequestDecision { } } +impl GptDiagnosticsRequestDecision { + /// An active decision, for tests in other modules that need one. + /// + /// The fields are private and built by `prepare_request` from a cookie or query + /// parameter; there is no other way to obtain an active decision across a module + /// boundary. + #[cfg(test)] + #[must_use] + pub(crate) fn active_for_tests() -> Self { + Self { + active: true, + clean_browser_path_and_query: None, + cookie_action: GptDiagnosticsCookieAction::None, + } + } +} + +#[cfg(test)] +mod head_seam_invariant_tests { + use super::*; + + /// Every combination of the three fields the decision carries. + fn all_decisions() -> Vec { + let mut out = Vec::new(); + for active in [false, true] { + for clean in [None, Some("/clean".to_string())] { + for cookie_action in [ + GptDiagnosticsCookieAction::None, + GptDiagnosticsCookieAction::SetSession, + GptDiagnosticsCookieAction::ClearSession, + ] { + out.push(GptDiagnosticsRequestDecision { + active, + clean_browser_path_and_query: clean.clone(), + cookie_action, + }); + } + } + } + out + } + + #[test] + fn requires_private_no_store_is_a_superset_of_injection() { + // Load-bearing relationship, not an incidental one. Whenever this decision + // injects anything into ``, the response must also be stamped + // `private, no-store` — which is what keeps request-scoped diagnostics out + // of a shared cache if the explicit assembly-mode gate in + // `create_html_stream_processor` is ever removed or bypassed. + // + // If a future change makes a script emit without also requiring the stamp, + // this fails here rather than silently in a cached template. + for decision in all_decisions() { + let injects = + decision.bootstrap_script().is_some() || decision.module_script_tag().is_some(); + if injects { + assert!( + decision.requires_private_no_store(), + "decision injects into but does not require private/no-store: \ + {decision:?}" + ); + } + } + } + + #[test] + fn a_default_decision_injects_nothing() { + let decision = GptDiagnosticsRequestDecision::default(); + assert_eq!(decision.bootstrap_script(), None); + assert_eq!(decision.module_script_tag(), None); + assert!( + !decision.requires_private_no_store(), + "an inert decision should not force the response private" + ); + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum QueryDirective { Absent, @@ -257,7 +334,7 @@ pub fn finalize_response( header::CACHE_CONTROL, HeaderValue::from_static("private, no-store"), ); - for name in CDN_CACHE_HEADERS { + for name in EDGE_CACHE_HEADER_NAMES { response.headers_mut().remove(*name); } } diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 4cc10f8da..c9b3f5ded 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -23,6 +23,7 @@ use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; use crate::auction::types::{ AuctionContext, AuctionRequest, AuctionResponse, Bid as AuctionBid, MediaType, }; +use crate::cache_policy::{CacheControlPolicy, EdgeCacheHeader}; use crate::consent_config::ConsentForwardingMode; use crate::cookies::{CONSENT_COOKIE_NAMES, strip_cookies}; use crate::error::TrustedServerError; @@ -754,14 +755,16 @@ impl PrebidIntegration { ) -> Result, Report> { let body = "// Script overridden by Trusted Server\n"; - http::Response::builder() + let mut response = http::Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, PREBID_BUNDLE_CONTENT_TYPE) - .header(header::CACHE_CONTROL, "public, max-age=31536000") .body(EdgeBody::from(body)) .change_context(TrustedServerError::Prebid { message: "Failed to build Prebid script handler response".to_string(), - }) + })?; + CacheControlPolicy::NoStorePrivate + .apply_to_headers(response.headers_mut(), EdgeCacheHeader::None); + Ok(response) } fn external_bundle_script_src(&self) -> String { @@ -3557,7 +3560,14 @@ external_bundle_sri = "sha384-AAAA" .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()) .expect("should have cache-control"); - assert!(cache_control.contains("max-age=31536000")); + assert_eq!( + cache_control, "no-store, private", + "neutralized stable shim must not be cached for a year" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "neutralized shim must not emit edge-cache headers" + ); let body = String::from_utf8( response diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 16cbac868..35c25ed20 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -575,6 +575,11 @@ pub trait IntegrationHeadInjector: Send + Sync { fn integration_id(&self) -> &'static str; /// Return HTML snippets to insert at the start of ``. fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec; + + /// Return attributes to add to the publisher TSJS bundle tag. + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + Vec::new() + } } /// Registration payload returned by integration builders. @@ -1053,6 +1058,16 @@ impl IntegrationRegistry { inserts } + /// Collect static attributes for the publisher TSJS bundle tag. + #[must_use] + pub fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + self.inner + .head_injectors + .iter() + .flat_map(|injector| injector.tsjs_script_tag_attributes()) + .collect() + } + /// Provide a snapshot of registered integrations and their hooks. #[must_use] pub fn registered_integrations(&self) -> Vec { @@ -1134,6 +1149,13 @@ impl IntegrationRegistry { ids } + /// Return whether an integration is enabled, including integrations whose + /// JavaScript is delivered outside the standard module bundles. + #[must_use] + pub fn is_enabled(&self, integration_id: &str) -> bool { + self.inner.enabled_integration_ids.contains(&integration_id) + } + /// Return JS module IDs for the main (synchronous) bundle, excluding /// modules registered with [`with_deferred_js`](IntegrationRegistrationBuilder::with_deferred_js). #[must_use] @@ -1321,6 +1343,58 @@ mod tests { use crate::platform::test_support::noop_services; use http::{HeaderValue, StatusCode, header}; + struct DefaultMetadataHeadInjector; + + impl IntegrationHeadInjector for DefaultMetadataHeadInjector { + fn integration_id(&self) -> &'static str { + "default-metadata" + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + Vec::new() + } + } + + struct StaticMetadataHeadInjector; + + impl IntegrationHeadInjector for StaticMetadataHeadInjector { + fn integration_id(&self) -> &'static str { + "static-metadata" + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + Vec::new() + } + + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + vec![ + ("data-ts-gam-attribution", "true"), + ("data-test-order", "second"), + ] + } + } + + #[test] + fn tsjs_script_tag_attributes_preserve_registration_order_and_default_empty() { + let registry = IntegrationRegistry::from_rewriters_with_head_injectors( + Vec::new(), + Vec::new(), + vec![ + Arc::new(DefaultMetadataHeadInjector), + Arc::new(StaticMetadataHeadInjector), + ], + ); + + assert_eq!( + registry.tsjs_script_tag_attributes(), + vec![ + ("data-ts-gam-attribution", "true"), + ("data-test-order", "second"), + ], + "should omit default-empty metadata and preserve registered attribute order" + ); + } + // Mock integration proxy for testing struct MockProxy; diff --git a/crates/trusted-server-core/src/integrations/testlight.rs b/crates/trusted-server-core/src/integrations/testlight.rs index 888427e52..80b2c4dfa 100644 --- a/crates/trusted-server-core/src/integrations/testlight.rs +++ b/crates/trusted-server-core/src/integrations/testlight.rs @@ -264,8 +264,9 @@ fn default_timeout_ms() -> u32 { } fn default_shim_src() -> String { - // Testlight is included in the unified bundle, so we return the unified script source. - // Uses conservative all-module hash since the registry is unavailable at config time. + // Testlight is included in the unified bundle, so return the registry-free + // unified script source. It intentionally omits `?v=` because the exact + // enabled module set is unavailable at config-default time. tsjs::tsjs_unified_script_src() } diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 70a4d6cfd..48e92faed 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -35,6 +35,7 @@ pub(crate) mod asset_image_optimizer; pub mod auction; pub mod auction_config_types; pub mod auth; +pub mod cache_policy; pub mod config; pub mod config_payload; pub mod consent; diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 287f1accf..7c80f9e12 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -12,6 +12,7 @@ //! - [`PlatformBackend`] — dynamic backend registration //! - [`PlatformHttpClient`] — outbound HTTP client //! - [`PlatformGeo`] — geographic information lookup +//! - [`PlatformTemplateAssembler`] — cold-response shared-template assembly //! //! ## Platform-Agnostic Components //! @@ -36,6 +37,8 @@ mod error; mod http; mod image_optimizer; mod kv; +mod template_assembly; +mod template_cache; #[cfg(test)] pub(crate) mod test_support; mod traits; @@ -52,6 +55,16 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; +pub use template_assembly::{ + PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, +}; +pub use template_cache::REPLAYABLE_POLICY_HEADERS; +pub use template_cache::{ + PlatformTemplateCache, PlatformTemplateCacheReservation, TEMPLATE_SCHEMA_VERSION, + TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, + TemplateCacheReservation, TemplateEntry, TemplateMetadata, UnavailableTemplateCache, + VaryHeaderValues, VarySpec, +}; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, diff --git a/crates/trusted-server-core/src/platform/template_assembly.rs b/crates/trusted-server-core/src/platform/template_assembly.rs new file mode 100644 index 000000000..441e7ca31 --- /dev/null +++ b/crates/trusted-server-core/src/platform/template_assembly.rs @@ -0,0 +1,77 @@ +//! Platform boundary for assembling a shared template with reader-specific state. +//! +//! Core owns the cache-safety ordering and the portable byte-seam fallback. An adapter +//! may provide a richer assembler for the cold response after the reader-neutral +//! template has been stored. + +use core::fmt; + +/// Why a platform assembler could not produce a document. +#[derive(Debug, derive_more::Display)] +pub enum TemplateAssemblyError { + /// The adapter has no template assembler. + #[display("this adapter cannot assemble shared templates")] + Unsupported, + /// The platform assembler rejected or could not process the document. + #[display("template assembly failed: {message}")] + Failed { + /// Human-readable failure context. + message: String, + }, +} + +impl core::error::Error for TemplateAssemblyError {} + +/// Assembles reader-specific state into a shared HTML template. +pub trait PlatformTemplateAssembler: Send + Sync { + /// Produce the complete document served to this reader. + /// + /// # Errors + /// + /// Returns [`TemplateAssemblyError`] when the adapter cannot assemble the template. + fn assemble(&self, template: &[u8], fragment: &[u8]) -> Result, TemplateAssemblyError>; +} + +impl fmt::Debug for dyn PlatformTemplateAssembler { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformTemplateAssembler") + } +} + +/// Default assembler used by adapters that do not provide platform assembly. +#[derive(Debug, Default, Clone, Copy)] +pub struct UnavailableTemplateAssembler; + +impl PlatformTemplateAssembler for UnavailableTemplateAssembler { + fn assemble( + &self, + _template: &[u8], + _fragment: &[u8], + ) -> Result, TemplateAssemblyError> { + Err(TemplateAssemblyError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unavailable_assembler_refuses_the_document() { + let error = UnavailableTemplateAssembler + .assemble(b"", b"") + .expect_err("should refuse when platform assembly is unavailable"); + + assert!(matches!(error, TemplateAssemblyError::Unsupported)); + } + + #[test] + fn assembler_contract_is_object_safe() { + let assembler: Box = Box::new(UnavailableTemplateAssembler); + + assert!(matches!( + assembler.assemble(b"template", b"fragment"), + Err(TemplateAssemblyError::Unsupported) + )); + } +} diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs new file mode 100644 index 000000000..ac3761b0f --- /dev/null +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -0,0 +1,1096 @@ +//! The shared transformed-template cache (C2) for the #1009 ESI validation spike. +//! +//! Three caches are in play and conflating them is what produced the original wrong +//! conclusion in the design doc, so this module names which one it is: +//! +//! | Cache | Contents | Owner | +//! | ----- | --------------------------------- | ------------------------------ | +//! | C1 | raw origin bytes | Fastly read-through. Not this. | +//! | C2 | post-`lol_html`, pre-assembly | **This module.** | +//! | C3 | final per-user assembled response | **Must never exist.** | +//! +//! C2 holds a *shared template*: no per-user bytes, and no decisions that depend on +//! the request. What may and may not live in it is +//! [§6.7 of the design doc](../../../../docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md), +//! and the invariant is enforced by the rendered-document byte-identity tests in +//! `publisher`. +//! +//! Spike-only. Remove with the spike. + +use core::fmt; +use std::collections::HashSet; + +use crate::creative_opportunities::AssemblyMode; + +/// Version of the transform that produced a cached template. +/// +/// Bump on **any** change to what the transform emits. Without it a deploy reads +/// yesterday's template shape and assembles against markers that moved, which fails +/// as a rendering bug far from its cause rather than as a cache miss. +/// +/// | Version | Transform | +/// | ------- | --------- | +/// | 1 | `` seam used an executable ESI include tag targeting the old fragment endpoint | +/// | 2 | Marker became the inert comment ``; the seam hands slots to `scheduleInitialAdInit` instead of assigning them | +/// | 3 | Marker became ``; canonical collision-safe key, explicit origin freshness, and complete repeated document-policy metadata | +/// | 4 | Marker is the shorter, accurate [`AD_ASSEMBLY_SEAM`](crate::publisher::AD_ASSEMBLY_SEAM) | +pub const TEMPLATE_SCHEMA_VERSION: u32 = 4; + +/// Inputs that select one cached template. +/// +/// Every field changes the emitted bytes for the same URL. A signal that changes the +/// bytes and is **not** here produces cross-served templates; a signal that is +/// per-user does not belong here at all — it belongs out of the template entirely. +/// That distinction is the whole design: the key holds per-*variant* signals, and +/// per-*user* signals are excluded from the template rather than keyed on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateCacheKey { + /// Full request URL, stated explicitly rather than inherited from an ambient + /// request, so the key cannot silently depend on what the caller happened to + /// mutate first. + pub url: String, + /// Host and scheme. The post-processed output is host-dependent by construction: + /// both reach `IntegrationHtmlContext` and drive URL rewriting. + pub request_host: String, + /// See [`Self::request_host`]. + pub request_scheme: String, + /// Publisher origin identity, including the outbound Host override. Two virtual + /// hosts can share a connection target while producing unrelated documents. + pub origin_identity: String, + /// Inline and ESI modes emit different template bytes. Without this they poison + /// each other's entries. + pub assembly_mode: AssemblyMode, + /// Values of the request headers the **origin** declares it varies on, in the + /// order the origin listed them. Not a fixed list: the origin is authoritative, + /// and hard-coding one here would silently drift when the origin's changes. + pub vary_values: Vec, + /// Digest of every setting that can shape the transformed template plus the tsjs + /// bundle. Over-invalidating is safe; omitting a shaping input cross-serves bytes. + pub template_fingerprint: String, + /// See [`TEMPLATE_SCHEMA_VERSION`]. + pub schema_version: u32, +} + +impl TemplateCacheKey { + /// Render a fixed-size opaque key for the platform cache. + /// + /// The canonical input is length-prefixed before hashing, so neither delimiters nor + /// raw request values can collide or leak into cache diagnostics. + #[must_use] + pub fn to_cache_key(&self) -> String { + use sha2::Digest as _; + + fn push(out: &mut Vec, part: &[u8]) { + out.extend_from_slice(&(part.len() as u64).to_be_bytes()); + out.extend_from_slice(part); + } + + let mut canonical = Vec::new(); + push(&mut canonical, b"ts-c2"); + push(&mut canonical, &self.schema_version.to_be_bytes()); + push( + &mut canonical, + match self.assembly_mode { + AssemblyMode::Inline => b"inline", + AssemblyMode::Esi => b"esi", + }, + ); + push(&mut canonical, self.request_scheme.as_bytes()); + push(&mut canonical, self.request_host.as_bytes()); + push(&mut canonical, self.origin_identity.as_bytes()); + push(&mut canonical, self.url.as_bytes()); + push(&mut canonical, self.template_fingerprint.as_bytes()); + push( + &mut canonical, + &(self.vary_values.len() as u64).to_be_bytes(), + ); + for varied in &self.vary_values { + push(&mut canonical, varied.name.as_bytes()); + match &varied.values { + None => push(&mut canonical, b"absent"), + Some(values) => { + push(&mut canonical, b"present"); + push(&mut canonical, &(values.len() as u64).to_be_bytes()); + for value in values { + push(&mut canonical, value); + } + } + } + } + + let digest = sha2::Sha256::digest(canonical); + format!("ts-c2-v{}-{}", self.schema_version, hex::encode(digest)) + } + + /// Surrogate keys to attach at insert, for purge-based rollback. + /// + /// `ts-template` purges every template at once, which is the rollback lever. + /// The per-URL key allows targeted invalidation. Both are needed: the broad one + /// for an incident, the narrow one for ordinary invalidation. + #[must_use] + pub fn surrogate_keys(&self) -> Vec { + vec!["ts-template".to_string(), self.url_surrogate_key()] + } + + /// Surrogate key for every variant of this publisher URL. + /// + /// Used to evict a malformed object without flushing unrelated article templates. + #[must_use] + pub fn url_surrogate_key(&self) -> String { + format!("ts-template-url-{}", digest_hex(self.url.as_bytes())) + } +} + +fn digest_hex(bytes: &[u8]) -> String { + use sha2::Digest as _; + hex::encode(sha2::Sha256::digest(bytes)) +} + +/// One configured `Vary` input exactly as it appeared on the request. +/// +/// `None` means absent. `Some(vec![vec![]])` means present with one empty field +/// value. Repeated fields stay separate and ordered; no UTF-8 conversion is involved. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VaryHeaderValues { + /// Validated, lowercase header name. + pub name: String, + /// Every raw field value in wire order, or `None` when absent. + pub values: Option>>, +} + +/// Origin response headers safe to store with a shared template and replay on a hit. +/// +/// Every one is a per-URL policy statement, identical for every reader. Nothing +/// per-reader (`Set-Cookie`) and nothing cache-controlling (`Cache-Control`, `ETag`, +/// `Surrogate-Control`) appears here, and it is an allowlist so a new origin header is +/// excluded until someone decides otherwise. +pub const REPLAYABLE_POLICY_HEADERS: &[&str] = &[ + "content-security-policy", + "content-security-policy-report-only", + "permissions-policy", + "referrer-policy", + "strict-transport-security", + "cross-origin-opener-policy", + "cross-origin-embedder-policy", + "cross-origin-resource-policy", + "origin-agent-cluster", + "reporting-endpoints", + "report-to", + "link", + "x-frame-options", + "x-content-type-options", + "content-language", + "x-robots-tag", +]; + +/// Headers the key covers by construction, whatever the operator configured. +/// +/// The shared path stores decoded identity bytes and negotiates the reader representation +/// only after assembly, so an origin declaring `Vary: Accept-Encoding` is covered without +/// reader input. This assumes those origin variants differ only by HTTP content coding; +/// operators must leave ESI disabled if an origin changes document semantics instead. +/// Without this carve-out, the ordinary declaration sent by any compressing origin reads +/// as an uncovered gap and disqualifies the response, so **C2 would never cache anything +/// against a real origin** unless the operator redundantly listed a header the transform +/// already normalizes. Found by review before it could make the spike measure a hit rate +/// of approximately zero and read that as a result. +const STRUCTURALLY_COVERED: &[&str] = &["accept-encoding"]; + +/// Request headers to include in the cache key, and where the list comes from. +/// +/// # The chicken-and-egg this resolves +/// +/// The key must cover everything the origin varies on, or two requests needing +/// different templates share one entry. But a **lookup happens before the fetch**, +/// so on a cold key the origin's `Vary` is not yet known. +/// +/// Three ways out, and the trade-off is real: +/// +/// 1. **Configure the list** — what this does. One lookup, no extra round trip, and +/// the operator states what the origin varies on. Cost: it drifts silently if the +/// origin's `Vary` changes and nobody updates config. +/// 2. **Two-phase lookup** — fetch a URL-keyed record holding the last-seen `Vary`, +/// then key properly. Correct, but doubles the lookups on every request. +/// 3. **Store the list alongside** and re-key on mismatch. Same cost as (2) plus +/// complexity. +/// +/// (1) is chosen for the spike because Step A already measured the origin's actual +/// `Vary`, the origin response is checked for drift before storage, and the configured +/// template-cache ceiling bounds how long a newly introduced mismatch can survive. +/// **This is a spike-grade choice, not a production one** — see the drift guard below. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VarySpec { + /// Header names, lowercased, in a fixed order. + names: Vec, +} + +impl VarySpec { + /// Build from configured header names. + /// + /// # Panics + /// + /// Panics when a name is not a valid HTTP field name. Runtime configuration is + /// validated with [`Self::try_new`] before this constructor is used. + #[must_use] + pub fn new(names: impl IntoIterator) -> Self { + Self::try_new(names).expect("VarySpec names should be validated at configuration load") + } + + /// Build from configured names, validating and deduplicating them. + /// + /// # Errors + /// + /// Returns the offending name when it is not a valid HTTP field name. + pub fn try_new(names: impl IntoIterator) -> Result { + let mut seen = HashSet::new(); + let mut normalized = Vec::new(); + for raw in names { + let name = http::header::HeaderName::from_bytes(raw.as_bytes()) + .map_err(|_| raw.clone())? + .as_str() + .to_string(); + if STRUCTURALLY_COVERED.contains(&name.as_str()) { + continue; + } + if seen.insert(name.clone()) { + normalized.push(name); + } + } + Ok(Self { names: normalized }) + } + + /// Configured names, lowercased. + #[must_use] + pub fn names(&self) -> &[String] { + &self.names + } + + /// Extract the key inputs from a request's headers. + /// + /// A header the origin varies on but the request omits still contributes an + /// entry, with an empty value — otherwise "absent" and "present but empty" + /// would collide, and those are different requests to the origin. + #[must_use] + pub fn values_from(&self, headers: &http::HeaderMap) -> Vec { + self.names + .iter() + .map(|name| { + let values = headers.contains_key(name.as_str()).then(|| { + headers + .get_all(name.as_str()) + .iter() + .map(|value| value.as_bytes().to_vec()) + .collect() + }); + VaryHeaderValues { + name: name.clone(), + values, + } + }) + .collect() + } + + /// Whether the origin's declared `Vary` contains anything this spec omits. + /// + /// The drift guard for choice (1) above. Called **after** the origin responds, + /// when its `Vary` is finally known: if the origin varies on something the key + /// did not cover, the template just built is unsafe to store, because a request + /// differing only in that header would read it. + /// + /// Returns the uncovered names, so the caller can log precisely which config is + /// stale rather than reporting a generic refusal. + #[must_use] + pub fn uncovered_by<'a>(&self, origin_vary: impl IntoIterator) -> Vec { + origin_vary + .into_iter() + .flat_map(|value| value.split(',')) + .map(|name| name.trim().to_ascii_lowercase()) + .filter(|name| !name.is_empty() && name != "*") + .filter(|name| !STRUCTURALLY_COVERED.contains(&name.as_str())) + .filter(|name| !self.names.contains(name)) + .collect() + } +} + +/// Metadata stored alongside the template bytes. +/// +/// `cache::core` carries **no HTTP semantics** — status, headers, encoding and +/// revalidation are all the caller's. Rather than storing origin headers and +/// replaying them, store only what is needed to rebuild a response from scratch. +/// +/// That choice is deliberate and load-bearing: the publisher path forces +/// `private, no-store` and strips validators *after* the origin send, so replaying a +/// stored origin header would fight it. Rebuilding every header on a hit means no +/// origin header is ever replayed and the `Set-Cookie` privacy net stays trivially +/// safe. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateMetadata { + /// Encoding of the stored bytes. C2 writes only `identity`; retaining the field in + /// metadata makes corrupt or stale representations fail validation on read. + pub content_encoding: String, + /// Content type to rebuild the response with. + pub content_type: String, + /// Schema version the bytes were produced under. Checked on read: a mismatch is + /// a miss, not an error, so a rollback to an older binary degrades to + /// re-transforming rather than misassembling. + pub schema_version: u32, + /// Length of the template bytes as written. + /// + /// Guards against a partially written entry. `Transaction::insert` consumes the + /// transaction, so a write that fails part-way cannot cancel the insert — there + /// is no handle left to cancel it with. Recording the intended length and + /// checking it on read makes a truncated entry a miss instead of a silently + /// short template that would assemble into a broken page. + pub body_len: u64, + /// Origin response headers that are policy, not per-reader state. + /// + /// Reconstructing headers from scratch on a hit keeps origin `Set-Cookie` and caching + /// directives out of a shared cache — but it also dropped `Content-Security-Policy`, + /// framing protection and `Content-Language`, weakening the page. These are + /// per-URL and identical for every reader, so they belong with the template. + /// + /// Deliberately an allowlist: anything per-reader or cache-controlling is excluded by + /// construction rather than by remembering to strip it. + pub policy_headers: Vec<(String, String)>, +} + +impl TemplateMetadata { + /// Serialize for `user_metadata`. Deliberately a tiny hand-rolled format rather + /// than JSON — one allocation, no dependency, and a parse failure is + /// unambiguous. + #[must_use] + pub fn encode(&self) -> Vec { + let mut out = format!( + "v={}\nce={}\nct={}\nlen={}", + self.schema_version, self.content_encoding, self.content_type, self.body_len + ); + for (name, value) in &self.policy_headers { + // Header values cannot contain newlines (the HTTP parser rejects them), so a + // newline-delimited encoding cannot be broken by a header value. + out.push_str(&format!("\nh={name}:{value}")); + } + out.into_bytes() + } + + /// Parse `user_metadata`. Returns `None` on anything unexpected, which callers + /// must treat as a cache miss. + #[must_use] + pub fn decode(raw: &[u8]) -> Option { + let text = core::str::from_utf8(raw).ok()?; + let mut schema_version = None; + let mut policy_headers = Vec::new(); + let mut content_encoding = None; + let mut content_type = None; + let mut body_len = None; + for line in text.lines() { + let (key, value) = line.split_once('=')?; + match key { + "v" => { + if schema_version.replace(value.parse().ok()?).is_some() { + return None; + } + } + "ce" => { + if content_encoding.replace(value.to_string()).is_some() { + return None; + } + } + "h" => { + let (name, header_value) = value.split_once(':')?; + let name = http::header::HeaderName::from_bytes(name.as_bytes()).ok()?; + if !REPLAYABLE_POLICY_HEADERS.contains(&name.as_str()) { + return None; + } + http::HeaderValue::from_bytes(header_value.as_bytes()).ok()?; + policy_headers.push((name.as_str().to_string(), header_value.to_string())); + } + "ct" => { + if content_type.replace(value.to_string()).is_some() { + return None; + } + } + "len" => { + if body_len.replace(value.parse().ok()?).is_some() { + return None; + } + } + _ => return None, + } + } + let content_encoding = content_encoding?; + // Every template is decoded before insert. Accepting another value here would + // let corrupt metadata label plaintext bytes as gzip on a warm hit. + if content_encoding != "identity" { + return None; + } + let content_type = content_type?; + http::HeaderValue::from_bytes(content_type.as_bytes()).ok()?; + if !content_type + .split(';') + .next() + .is_some_and(|media_type| media_type.trim().eq_ignore_ascii_case("text/html")) + { + return None; + } + Some(Self { + schema_version: schema_version?, + policy_headers, + content_encoding, + content_type, + body_len: body_len?, + }) + } +} + +/// Why a template read did not produce usable bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum TemplateCacheMiss { + /// No entry for this key. + #[display("no cached template for this key")] + NotFound, + /// Found, but produced by a different transform version. + #[display("cached template has a different schema version")] + SchemaMismatch, + /// Found, but its metadata could not be parsed. + #[display("cached template metadata is unreadable")] + UnreadableMetadata, + /// Found, but shorter than the metadata says it should be — a write that failed + /// part-way. See [`TemplateMetadata::body_len`]. + #[display("cached template is truncated")] + Truncated, + /// This platform has no template cache. + #[display("no template cache on this platform")] + Unsupported, +} + +impl core::error::Error for TemplateCacheMiss {} + +/// Errors a template cache write can produce. +#[derive(Debug, derive_more::Display)] +pub enum TemplateCacheError { + /// This platform has no template cache. + #[display("no template cache on this platform")] + Unsupported, + /// The platform rejected the operation. + #[display("template cache backend error: {message}")] + Backend { + /// What the backend reported. + message: String, + }, +} + +impl core::error::Error for TemplateCacheError {} + +/// Result of the pre-origin cache transaction. +pub enum TemplateCacheLookup { + /// A fresh usable template. + Hit(TemplateEntry), + /// This request owns the obligation to provide or cancel the cold object. + Reserved(TemplateCacheReservation), + /// This adapter deliberately has no shared-template cache. + Unsupported, + /// A cache object existed but failed schema, metadata, or length validation. + Invalid(TemplateCacheMiss), +} + +/// Platform-owned insert obligation. Dropping it cancels, making every early-return +/// path safe without an async cleanup ladder in the publisher pipeline. +pub struct TemplateCacheReservation { + inner: Option>, +} + +impl core::fmt::Debug for TemplateCacheReservation { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TemplateCacheReservation") + .finish_non_exhaustive() + } +} + +impl TemplateCacheReservation { + /// Wrap a platform reservation. + #[must_use] + pub fn new(inner: Box) -> Self { + Self { inner: Some(inner) } + } + + /// Fulfil the reservation with a validated template. + /// + /// # Errors + /// + /// Returns the platform cache error when the reservation cannot be fulfilled. + pub fn insert( + mut self, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + self.inner + .take() + .ok_or_else(|| TemplateCacheError::Backend { + message: "template reservation was already consumed".to_string(), + })? + .insert(metadata, body, max_age) + } + + /// Explicitly give up the reservation. Drop performs the same operation as a net. + /// + /// # Errors + /// + /// Returns the platform cache error when the reservation cannot be cancelled. + pub fn cancel(mut self) -> Result<(), TemplateCacheError> { + self.inner + .take() + .ok_or_else(|| TemplateCacheError::Backend { + message: "template reservation was already consumed".to_string(), + })? + .cancel() + } +} + +impl Drop for TemplateCacheReservation { + fn drop(&mut self) { + if let Some(inner) = self.inner.take() + && let Err(err) = inner.cancel() + { + log::warn!("c2_template_cache reservation cancellation failed: {err}"); + } + } +} + +/// Adapter-specific ownership token returned by a transactional lookup. +pub trait PlatformTemplateCacheReservation: Send { + /// Insert and discharge the obligation. + /// + /// # Errors + /// + /// Returns an adapter-specific cache error when the insert fails. + fn insert( + self: Box, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError>; + + /// Cancel and allow a waiting request to take ownership. + /// + /// # Errors + /// + /// Returns an adapter-specific cache error when cancellation fails. + fn cancel(self: Box) -> Result<(), TemplateCacheError>; +} + +impl fmt::Debug for dyn PlatformTemplateCache { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformTemplateCache") + } +} + +/// A platform's shared-template cache. +/// +/// Only the Fastly adapter implements this; every other adapter uses +/// [`UnavailableTemplateCache`], which reports [`TemplateCacheMiss::Unsupported`] so +/// the caller transforms every time rather than failing. +/// +/// `Send + Sync` on the trait, `?Send` on the futures: `RuntimeServices` is held in a +/// `LazyLock` static, so the trait object must cross threads even though the futures +/// themselves never do — the platform layer is `!Send` by construction. +#[async_trait::async_trait(?Send)] +pub trait PlatformTemplateCache: Send + Sync { + /// Transactionally look up a template before origin work begins. + async fn lookup_or_reserve( + &self, + key: &TemplateCacheKey, + ) -> Result { + Ok(match self.get(key).await { + Ok(entry) => TemplateCacheLookup::Hit(entry), + Err(TemplateCacheMiss::Unsupported | TemplateCacheMiss::NotFound) => { + TemplateCacheLookup::Unsupported + } + Err(miss) => TemplateCacheLookup::Invalid(miss), + }) + } + + /// Read a template. `Err` is a miss, not a failure — every variant means + /// "transform it yourself". + async fn get(&self, key: &TemplateCacheKey) -> Result; + + /// Store a template. + /// + /// Callers must not call this without having consulted the C2 eligibility gate + /// first: this method stores what it is given and cannot tell a shared template + /// from a per-user one. + async fn put( + &self, + key: &TemplateCacheKey, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError>; + + /// Purge every cached variant for one publisher URL. + async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError>; + + /// Purge every stored template. The rollback lever. + async fn purge_all(&self) -> Result<(), TemplateCacheError>; +} + +/// A template read from the cache. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateEntry { + /// Metadata stored at insert. + pub metadata: TemplateMetadata, + /// The transformed template bytes. + pub body: Vec, +} + +/// The null object, used by every adapter without a template cache. +/// +/// Reporting [`TemplateCacheMiss::Unsupported`] rather than erroring means the +/// ESI assembly mode degrades to transforming per request on Cloudflare, Axum and Spin +/// instead of failing — the mode stays portable, only the caching is not. +pub struct UnavailableTemplateCache; + +#[async_trait::async_trait(?Send)] +impl PlatformTemplateCache for UnavailableTemplateCache { + async fn lookup_or_reserve( + &self, + _key: &TemplateCacheKey, + ) -> Result { + Ok(TemplateCacheLookup::Unsupported) + } + + async fn get(&self, _key: &TemplateCacheKey) -> Result { + Err(TemplateCacheMiss::Unsupported) + } + + async fn put( + &self, + _key: &TemplateCacheKey, + _metadata: &TemplateMetadata, + _body: Vec, + _max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } + + async fn purge_url(&self, _key: &TemplateCacheKey) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } + + async fn purge_all(&self) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn key() -> TemplateCacheKey { + TemplateCacheKey { + url: "https://example.com/news/article".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + origin_identity: "https://origin.example.com\0origin.example.com".to_string(), + assembly_mode: AssemblyMode::Esi, + vary_values: vec![VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }], + template_fingerprint: "abc123".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + } + } + + struct CountingReservation(Arc); + + impl PlatformTemplateCacheReservation for CountingReservation { + fn insert( + self: Box, + _metadata: &TemplateMetadata, + _body: Vec, + _max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + Ok(()) + } + + fn cancel(self: Box) -> Result<(), TemplateCacheError> { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + + #[test] + fn dropping_an_unfulfilled_reservation_cancels_exactly_once() { + let cancellations = Arc::new(AtomicUsize::new(0)); + drop(TemplateCacheReservation::new(Box::new( + CountingReservation(Arc::clone(&cancellations)), + ))); + assert_eq!(cancellations.load(Ordering::SeqCst), 1); + } + + /// Every field must change the key. A field that does not is a cross-serving + /// bug: two requests needing different templates would share one entry. + #[test] + fn every_field_changes_the_key() { + let base = key().to_cache_key(); + + let mut mode = key(); + mode.assembly_mode = AssemblyMode::Inline; + assert_ne!( + mode.to_cache_key(), + base, + "assembly mode must change the key" + ); + + let mut url = key(); + url.url = "https://example.com/other".to_string(); + assert_ne!(url.to_cache_key(), base, "url must change the key"); + + let mut host = key(); + host.request_host = "other.example.com".to_string(); + assert_ne!(host.to_cache_key(), base, "host must change the key"); + + let mut scheme = key(); + scheme.request_scheme = "http".to_string(); + assert_ne!(scheme.to_cache_key(), base, "scheme must change the key"); + + let mut origin = key(); + origin.origin_identity = "https://origin.example.com\0other.example.com".to_string(); + assert_ne!( + origin.to_cache_key(), + base, + "origin Host identity must change the key" + ); + + let mut fingerprint = key(); + fingerprint.template_fingerprint = "def456".to_string(); + assert_ne!( + fingerprint.to_cache_key(), + base, + "template fingerprint must change the key" + ); + + let mut schema = key(); + schema.schema_version = TEMPLATE_SCHEMA_VERSION + 1; + assert_ne!( + schema.to_cache_key(), + base, + "schema version must change the key" + ); + + let mut vary = key(); + vary.vary_values = vec![VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"0".to_vec()]), + }]; + assert_ne!(vary.to_cache_key(), base, "vary values must change the key"); + } + + /// The reason for length prefixes rather than a delimiter. + #[test] + fn values_containing_delimiters_cannot_collide() { + let mut a = key(); + a.request_host = "a".to_string(); + a.url = "b:c".to_string(); + + let mut b = key(); + b.request_host = "a:b".to_string(); + b.url = "c".to_string(); + + assert_ne!( + a.to_cache_key(), + b.to_cache_key(), + "field values containing the delimiter must not produce the same key; a \ + collision here serves one visitor's template to another" + ); + } + + #[test] + fn rendered_key_is_fixed_size_and_contains_no_request_material() { + let rendered = key().to_cache_key(); + assert_eq!(rendered.len(), "ts-c2-v3-".len() + 64); + for sensitive in ["example.com", "/news/article", "rsc", "abc123"] { + assert!( + !rendered.contains(sensitive), + "key leaked `{sensitive}`: {rendered}" + ); + } + } + + #[test] + fn vary_header_names_are_matched_case_insensitively() { + let mut upper = key(); + upper.vary_values = vec![VaryHeaderValues { + name: "RSC".to_ascii_lowercase(), + values: Some(vec![b"1".to_vec()]), + }]; + assert_eq!( + upper.to_cache_key(), + key().to_cache_key(), + "header names are case-insensitive, so casing must not split the cache" + ); + } + + #[test] + fn vary_values_are_order_sensitive() { + // The origin lists them in a fixed order and the caller preserves it, so a + // differing order means differing inputs rather than the same request. + let mut a = key(); + a.vary_values = vec![ + VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }, + VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"article".to_vec()]), + }, + ]; + let mut b = key(); + b.vary_values = vec![ + VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"article".to_vec()]), + }, + VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }, + ]; + assert_ne!(a.to_cache_key(), b.to_cache_key()); + } + + #[test] + fn surrogate_keys_carry_a_global_and_a_per_url_lever() { + let keys = key().surrogate_keys(); + assert!( + keys.contains(&"ts-template".to_string()), + "a global purge lever is what makes rollback possible" + ); + assert_eq!(keys.len(), 2, "global plus per-URL"); + assert!( + !keys[1].contains(char::is_whitespace), + "surrogate keys are space-delimited; whitespace would purge more than \ + intended, got {:?}", + keys[1] + ); + assert!( + !keys[1].contains('/') && !keys[1].contains(':'), + "URL punctuation must be reduced, got {:?}", + keys[1] + ); + } + + #[test] + fn punctuation_distinct_urls_have_distinct_surrogate_keys() { + let mut slash = key(); + slash.url = "https://example.com/a/b".to_string(); + let mut colon = key(); + colon.url = "https://example.com/a:b".to_string(); + assert_ne!(slash.surrogate_keys()[1], colon.surrogate_keys()[1]); + } + + #[test] + fn an_absent_vary_header_is_distinct_from_an_empty_one() { + // "absent" and "present but empty" are different requests to the origin, so + // they must not share a template. + let spec = VarySpec::new(["RSC".to_string()]); + let absent_headers = http::HeaderMap::new(); + let absent = spec.values_from(&absent_headers); + let mut empty_headers = http::HeaderMap::new(); + empty_headers.insert("rsc", http::HeaderValue::from_static("")); + let empty = spec.values_from(&empty_headers); + assert_ne!(absent, empty); + + // The distinction that does matter: a present value differs from both. + let mut present_headers = http::HeaderMap::new(); + present_headers.insert("rsc", http::HeaderValue::from_static("1")); + let present = spec.values_from(&present_headers); + assert_ne!(present, absent); + } + + #[test] + fn repeated_and_non_utf8_vary_values_are_preserved() { + let spec = VarySpec::new(["x-route".to_string()]); + let mut headers = http::HeaderMap::new(); + headers.append("x-route", http::HeaderValue::from_static("first")); + headers.append( + "x-route", + http::HeaderValue::from_bytes(b"\xffsecond").expect("obs-text is valid field data"), + ); + assert_eq!( + spec.values_from(&headers), + vec![VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"first".to_vec(), b"\xffsecond".to_vec()]), + }] + ); + } + + #[test] + fn vary_spec_lowercases_configured_names() { + assert_eq!( + VarySpec::new(["RSC".to_string(), "Accept-Encoding".to_string()]).names(), + ["rsc"] + ); + } + + #[test] + fn vary_spec_rejects_invalid_names_and_deduplicates_case_insensitively() { + assert_eq!( + VarySpec::try_new(["not a header".to_string()]), + Err("not a header".to_string()) + ); + assert_eq!( + VarySpec::try_new(["RSC".to_string(), "rsc".to_string()]) + .expect("valid names") + .names(), + ["rsc"] + ); + } + + #[test] + fn drift_is_detected_when_the_origin_varies_on_something_unconfigured() { + // The failure mode configured-Vary has: the origin adds a header to its Vary, + // nobody updates config, and requests differing only in that header start + // sharing a template. + let spec = VarySpec::new(["rsc".to_string()]); + + assert!( + spec.uncovered_by(["rsc"]).is_empty(), + "a fully covered Vary is not drift" + ); + assert_eq!( + spec.uncovered_by(["rsc, next-router-prefetch, Accept-Encoding"]), + vec!["next-router-prefetch"], + "uncovered names must be reported so the stale config is identifiable; \ + accept-encoding is excluded because the key covers it structurally" + ); + } + + #[test] + fn a_key_field_counts_as_coverage_without_being_configured() { + // The failure this prevents is silent and total: every compressing origin sends + // `Vary: Accept-Encoding`, so treating it as a gap means the cache never stores + // anything, and a spike measuring hit rate would report ~0 and look like a + // finding rather than a bug. + let spec = VarySpec::new([]); + + assert!( + spec.uncovered_by(["Accept-Encoding"]).is_empty(), + "the shared path uses one upstream encoding offer and stores identity bytes" + ); + assert_eq!( + spec.uncovered_by(["accept-encoding, rsc"]), + vec!["rsc"], + "only the genuinely uncovered name should be reported" + ); + } + + #[test] + fn a_wildcard_vary_is_not_reported_as_a_named_gap() { + // `Vary: *` means uncacheable, which the eligibility gate handles. Reporting + // it here would produce a nonsense "configure a header called *". + let spec = VarySpec::new(["rsc".to_string()]); + assert!(spec.uncovered_by(["*"]).is_empty()); + } + + #[test] + fn metadata_round_trips() { + let metadata = TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: vec![ + ( + "content-security-policy".to_string(), + "default-src 'self'".to_string(), + ), + ( + "content-security-policy".to_string(), + "script-src 'self'".to_string(), + ), + ( + "link".to_string(), + "; rel=preload; as=script".to_string(), + ), + ], + content_type: "text/html; charset=utf-8".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 42, + }; + let decoded = + TemplateMetadata::decode(&metadata.encode()).expect("should decode what it encoded"); + assert_eq!(decoded, metadata); + } + + #[test] + fn unparseable_metadata_is_a_miss_not_a_panic() { + for raw in [ + &b"not-key-value"[..], + &b"v=notanumber\nce=gzip\nct=text/html\nlen=1"[..], + &b"v=1\nce=gzip\nct=text/html"[..], + &b"v=1\nce=gzip\nct=text/html\nlen=1\nunexpected=1"[..], + &b"v=1\nv=1\nce=identity\nct=text/html\nlen=1"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=cache-control:public"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=not-a-policy:value"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=malformed"[..], + &b"v=1\nce=identity\nct=application/json\nlen=1"[..], + &[0xff, 0xfe][..], + ] { + assert_eq!( + TemplateMetadata::decode(raw), + None, + "malformed metadata must be a miss, not a partial read: {raw:?}" + ); + } + } + + #[test] + fn the_policy_allowlist_covers_document_security_and_delivery_headers() { + for required in [ + "strict-transport-security", + "cross-origin-opener-policy", + "cross-origin-embedder-policy", + "cross-origin-resource-policy", + "origin-agent-cluster", + "reporting-endpoints", + "report-to", + "link", + ] { + assert!( + REPLAYABLE_POLICY_HEADERS.contains(&required), + "warm ESI hits must preserve {required}" + ); + } + } + + #[tokio::test] + async fn the_null_object_reports_unsupported_rather_than_failing() { + // Degrading to per-request transformation keeps the shared modes portable on + // adapters with no cache; erroring would make them Fastly-only outright. + let cache = UnavailableTemplateCache; + assert_eq!( + cache.get(&key()).await.err(), + Some(TemplateCacheMiss::Unsupported) + ); + assert!(matches!( + cache + .put( + &key(), + &TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: Vec::new(), + content_type: "text/html".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 0, + }, + Vec::new(), + std::time::Duration::from_secs(1) + ) + .await, + Err(TemplateCacheError::Unsupported) + )); + } +} diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index a39a26430..e9e02e524 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -168,6 +168,16 @@ pub struct RuntimeServices { /// per-request basis by cloning [`RuntimeServices`] with /// [`RuntimeServices::with_kv_store`]. pub(crate) kv_store: Arc, + /// Shared transformed-template cache (C2). Defaults to + /// [`UnavailableTemplateCache`], so adapters without one degrade to transforming + /// per request rather than failing. Spike-only; see + /// [`crate::platform::template_cache`]. + pub(crate) template_cache: Arc, + /// Platform-specific cold-response template assembler. + /// + /// Defaults to [`super::UnavailableTemplateAssembler`]. Core retains a portable + /// byte-seam fallback when this service is unavailable or rejects a document. + pub(crate) template_assembler: Arc, /// Dynamic backend registration and name prediction. pub(crate) backend: Arc, /// Outbound HTTP client abstraction. @@ -223,6 +233,18 @@ impl RuntimeServices { &*self.kv_store } + /// The shared transformed-template cache. Spike-only. + #[must_use] + pub fn template_cache(&self) -> &dyn super::PlatformTemplateCache { + &*self.template_cache + } + + /// Returns the platform-specific cold-response template assembler. + #[must_use] + pub fn template_assembler(&self) -> &dyn super::PlatformTemplateAssembler { + &*self.template_assembler + } + /// Returns the dynamic backend service. #[must_use] pub fn backend(&self) -> &dyn PlatformBackend { @@ -272,6 +294,29 @@ impl RuntimeServices { ..self } } + + /// Returns a clone of this instance with the template cache replaced. + /// + /// Spike-only (#1009). + #[must_use] + pub fn with_template_cache(self, cache: Arc) -> Self { + Self { + template_cache: cache, + ..self + } + } + + /// Returns a clone of this instance with the template assembler replaced. + #[must_use] + pub fn with_template_assembler( + self, + assembler: Arc, + ) -> Self { + Self { + template_assembler: assembler, + ..self + } + } } impl fmt::Debug for RuntimeServices { @@ -290,6 +335,8 @@ pub struct RuntimeServicesBuilder { config_store: Option>, secret_store: Option>, kv_store: Option>, + template_cache: Option>, + template_assembler: Option>, backend: Option>, http_client: Option>, geo: Option>, @@ -303,6 +350,8 @@ impl RuntimeServicesBuilder { config_store: None, secret_store: None, kv_store: None, + template_cache: None, + template_assembler: None, backend: None, http_client: None, geo: None, @@ -325,6 +374,23 @@ impl RuntimeServicesBuilder { self } + /// Set the shared transformed-template cache. Spike-only. + #[must_use] + pub fn template_cache(mut self, cache: Arc) -> Self { + self.template_cache = Some(cache); + self + } + + /// Set the platform-specific cold-response template assembler. + #[must_use] + pub fn template_assembler( + mut self, + assembler: Arc, + ) -> Self { + self.template_assembler = Some(assembler); + self + } + /// Set the KV store implementation. #[must_use] pub fn kv_store(mut self, kv_store: Arc) -> Self { @@ -387,6 +453,14 @@ impl RuntimeServicesBuilder { kv_store: self .kv_store .expect("should set kv_store before building RuntimeServices"), + // Defaulted rather than required: an adapter with no template cache + // should degrade to transforming per request, not fail to build. + template_cache: self + .template_cache + .unwrap_or_else(|| Arc::new(super::UnavailableTemplateCache)), + template_assembler: self + .template_assembler + .unwrap_or_else(|| Arc::new(super::UnavailableTemplateAssembler)), backend: self .backend .expect("should set backend before building RuntimeServices"), diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index ea0a0cf8d..2ad8091fd 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -13,6 +13,11 @@ use std::sync::{Arc, LazyLock, Mutex}; use std::time::Duration; use web_time::{SystemTime, UNIX_EPOCH}; +use crate::cache_policy::{ + CachePolicy, EdgeCacheHeader, NO_STORE_PRIVATE_CACHE_CONTROL, + apply_no_store_private_to_headers, cache_control_headers_are_private_or_no_store, + remove_edge_cache_headers, +}; use crate::constants::{ HEADER_ACCEPT, HEADER_ACCEPT_ENCODING, HEADER_ACCEPT_LANGUAGE, HEADER_REFERER, HEADER_USER_AGENT, HEADER_X_FORWARDED_FOR, @@ -96,7 +101,7 @@ const ASSET_PROXY_STRIP_RESPONSE_HEADERS: [&str; 3] = ["set-cookie", "strict-transport-security", "clear-site-data"]; /// Cache-control value used when asset proxy responses must not be stored. -pub const ASSET_NO_STORE_PRIVATE_CACHE_CONTROL: &str = "no-store, private"; +pub const ASSET_NO_STORE_PRIVATE_CACHE_CONTROL: &str = NO_STORE_PRIVATE_CACHE_CONTROL; /// Cache policy metadata emitted by the asset proxy handler. /// @@ -109,13 +114,27 @@ pub enum AssetProxyCachePolicy { OriginControlled, /// Reapply `Cache-Control: no-store, private` after standard finalization. NoStorePrivate, + /// Reapply an operator-selected normalized cache policy after finalization. + Normalized(CachePolicy), } impl AssetProxyCachePolicy { /// Apply protected cache headers after route-level response finalization. - pub fn apply_after_route_finalization(self, response: &mut Response) { - if self == Self::NoStorePrivate { - apply_no_store_cache_control(response); + pub fn apply_after_route_finalization( + self, + response: &mut Response, + edge_header: EdgeCacheHeader, + ) { + match self { + Self::OriginControlled => {} + Self::NoStorePrivate => apply_no_store_cache_control(response), + Self::Normalized(policy) => { + if cache_control_headers_are_private_or_no_store(response.headers()) { + remove_edge_cache_headers(response.headers_mut()); + } else { + policy.apply_to_headers(response.headers_mut(), edge_header); + } + } } } } @@ -169,6 +188,11 @@ impl AssetProxyResponse { apply_no_store_cache_control(&mut self.response); } + fn apply_normalized_cache_policy(&mut self, policy: CachePolicy) { + self.cache_policy = AssetProxyCachePolicy::Normalized(policy); + policy.apply_to_headers(self.response.headers_mut(), EdgeCacheHeader::None); + } + /// Return cache policy metadata for router finalization. #[must_use] pub fn cache_policy(&self) -> AssetProxyCachePolicy { @@ -1020,10 +1044,7 @@ fn strip_asset_proxy_response_headers(response: &mut Response) { } fn apply_no_store_cache_control(response: &mut Response) { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static(ASSET_NO_STORE_PRIVATE_CACHE_CONTROL), - ); + apply_no_store_private_to_headers(response.headers_mut()); } fn should_preflight_s3( @@ -1206,6 +1227,13 @@ pub async fn handle_asset_proxy_request( let mut response = platform_response_to_fastly_asset(platform_resp); strip_asset_proxy_response_headers(response.response_mut()); + let status = response.response().status(); + if (status.is_success() || status == StatusCode::NOT_MODIFIED) + && let Some(policy) = settings.asset_cache_policy_for_path(incoming_path)? + { + response.apply_normalized_cache_policy(policy); + } + Ok(response) } @@ -2167,6 +2195,7 @@ mod tests { use std::io; use std::rc::Rc; use std::sync::{Arc, Mutex}; + use std::time::Duration; use super::{ AssetProxyCachePolicy, IMAGE_FALLBACK_CONTENT_TYPE, ProxyRequestConfig, @@ -2177,6 +2206,7 @@ mod tests { proxy_request, rebuild_response_with_body, reconstruct_and_validate_signed_target, redirect_is_permitted, stream_asset_body, }; + use crate::cache_policy::{CachePolicy, EdgeCacheHeader}; use crate::constants::{HEADER_ACCEPT, HEADER_X_FORWARDED_FOR}; use crate::creative; use crate::error::{IntoHttpResponse, TrustedServerError}; @@ -2191,9 +2221,9 @@ mod tests { use crate::settings::{ AssetImageOptimizerConfig, AssetOriginAuth, ImageOptimizerAspectRatioConfig, ImageOptimizerCropOffsetsConfig, ImageOptimizerProfileSet, ImageOptimizerSettings, - OriginQueryPolicy, ProxyAssetRoute, S3SigV4AuthConfig, UnknownProfilePolicy, + OriginQueryPolicy, ProxyAssetRoute, S3SigV4AuthConfig, Settings, UnknownProfilePolicy, }; - use crate::test_support::tests::create_test_settings; + use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; use bytes::Bytes; use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::response_builder as edge_response_builder; @@ -4304,6 +4334,167 @@ mod tests { }); } + #[test] + fn handle_asset_proxy_request_replaces_third_party_cache_policy_for_rehosted_asset() { + futures::executor::block_on(async { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"asset".to_vec(), + vec![(header::CACHE_CONTROL.as_str(), "no-store")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let settings = Settings::from_toml(&format!( + r#"{} + + [[cache.asset_rules]] + id = "fingerprinted-assets" + enabled = true + path_globs = ["/assets/**/*.js"] + fingerprint_style = "hex" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + )) + .expect("should parse settings with cache asset rule"); + let req = build_http_request( + Method::GET, + "https://www.example.com/assets/app.0123abcd.js", + ); + let route = ProxyAssetRoute::new("/assets/", "https://assets.example.com"); + + let asset_response = handle_asset_proxy_request(&settings, &services, req, &route) + .await + .expect("should proxy asset request"); + assert_eq!( + asset_response.cache_policy(), + AssetProxyCachePolicy::Normalized(CachePolicy::public_immutable( + Duration::from_secs(31_536_000) + )), + "should carry normalized cache policy metadata" + ); + + let mut response = asset_response + .into_response() + .expect("should return buffered asset response"); + assert_eq!( + response_header(&response, header::CACHE_CONTROL), + Some("public, max-age=31536000, immutable"), + "configured rehost policy should replace the third-party no-store directive" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "runtime-specific edge header should wait for adapter finalization" + ); + + AssetProxyCachePolicy::Normalized(CachePolicy::public_immutable(Duration::from_secs( + 31_536_000, + ))) + .apply_after_route_finalization(&mut response, EdgeCacheHeader::SurrogateControl); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=31536000"), + "Fastly finalization should render Surrogate-Control" + ); + }); + } + + #[test] + fn normalized_asset_policy_preserves_final_private_or_no_store_directives() { + for cache_control in ["private, max-age=0", "no-store"] { + let mut response = edge_response_builder() + .header(header::CACHE_CONTROL, cache_control) + .header("surrogate-control", "max-age=31536000") + .header("cdn-cache-control", "max-age=31536000") + .header("cloudflare-cdn-cache-control", "max-age=31536000") + .body(EdgeBody::empty()) + .expect("should build asset response"); + + AssetProxyCachePolicy::Normalized(CachePolicy::public_immutable(Duration::from_secs( + 31_536_000, + ))) + .apply_after_route_finalization(&mut response, EdgeCacheHeader::SurrogateControl); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "final privacy directive should veto normalized cache policy" + ); + assert!( + [ + "surrogate-control", + "cdn-cache-control", + "cloudflare-cdn-cache-control", + ] + .iter() + .all(|name| !response.headers().contains_key(*name)), + "final privacy directive should remove every edge-cache header" + ); + } + } + + #[test] + fn handle_asset_proxy_request_leaves_non_matching_assets_origin_controlled() { + futures::executor::block_on(async { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"asset".to_vec(), + vec![(header::CACHE_CONTROL.as_str(), "public, max-age=60")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let settings = Settings::from_toml(&format!( + r#"{} + + [[cache.asset_rules]] + id = "fingerprinted-assets" + enabled = true + path_globs = ["/assets/**/*.js"] + fingerprint_style = "hex" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + )) + .expect("should parse settings with cache asset rule"); + let req = build_http_request(Method::GET, "https://www.example.com/assets/app.js"); + let route = ProxyAssetRoute::new("/assets/", "https://assets.example.com"); + + let asset_response = handle_asset_proxy_request(&settings, &services, req, &route) + .await + .expect("should proxy asset request"); + + assert_eq!( + asset_response.cache_policy(), + AssetProxyCachePolicy::OriginControlled, + "non-fingerprinted file should not receive normalized immutable policy" + ); + let response = asset_response + .into_response() + .expect("should return buffered asset response"); + assert_eq!( + response_header(&response, header::CACHE_CONTROL), + Some("public, max-age=60"), + "origin-controlled response should preserve origin cache header" + ); + }); + } + fn test_profile_set() -> ImageOptimizerProfileSet { let mut profiles = HashMap::new(); profiles.insert("default".to_string(), "width=1920".to_string()); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4bed98327..10bd1814d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -21,7 +21,7 @@ use std::borrow::Cow; use std::io::Write; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant, SystemTime}; use brotli::Decompressor; use brotli::enc::BrotliEncoderParams; @@ -48,20 +48,30 @@ use crate::auction::telemetry::{ use crate::auction::types::{ AuctionContext, AuctionRequest, Bid, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, }; +use crate::cache_policy::{ + CachePolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, +}; use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; +use crate::creative_opportunities::{AssemblyMode, CreativeOpportunitiesConfig}; use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; +use crate::html_processor::BodyCloseInjection; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; use crate::integrations::IntegrationRegistry; -use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use crate::platform::{ + GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, VarySpec, +}; use crate::price_bucket::{PriceGranularity, price_bucket}; -use crate::response_privacy::enforce_synthesized_html_cache_privacy; +use crate::response_privacy::{enforce_private_no_store, enforce_synthesized_html_cache_privacy}; use crate::rsc_flight::RscFlightUrlRewriter; -use crate::settings::Settings; +use crate::settings::{ + AUCTION_DEBUG_METADATA_ALLOWLIST, AUCTION_DEBUG_UPSTREAM_METADATA_KEYS, + AuctionDebugCommentFormat, AuctionDebugCommentOptions, AuctionDebugCommentVerbosity, Settings, +}; use crate::streaming_processor::{ BodyStreamDecoder, BodyStreamEncoder, Compression, GzipDecodeReader, PipelineConfig, STREAM_CHUNK_SIZE, StreamProcessor, StreamingPipeline, @@ -70,6 +80,68 @@ use crate::streaming_replacer::create_url_replacer; const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +const HEADER_X_TS_C2_CACHE: &str = "x-ts-c2-cache"; +const HEADER_X_TS_ASSEMBLY: &str = "x-ts-assembly"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum C2ResponseState { + Hit, + MissReserved, + MissStored, + MissStoreError, + BypassRequest, + BypassResponse, + Unsupported, + Invalid, + BackendError, +} + +impl C2ResponseState { + const fn as_str(self) -> &'static str { + match self { + Self::Hit => "hit", + Self::MissReserved => "miss-reserved", + Self::MissStored => "miss-stored", + Self::MissStoreError => "miss-store-error", + Self::BypassRequest => "bypass-request", + Self::BypassResponse => "bypass-response", + Self::Unsupported => "unsupported", + Self::Invalid => "invalid", + Self::BackendError => "backend-error", + } + } +} + +fn set_c2_response_state(response: &mut Response, state: C2ResponseState) { + response.headers_mut().insert( + HEADER_X_TS_C2_CACHE, + HeaderValue::from_static(state.as_str()), + ); +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum AssemblyResponseState { + EsiParser, + ByteSeamFallback, + ByteSeam, +} + +impl AssemblyResponseState { + const fn as_str(self) -> &'static str { + match self { + Self::EsiParser => "esi-parser", + Self::ByteSeamFallback => "byte-seam-fallback", + Self::ByteSeam => "byte-seam", + } + } +} + +fn set_assembly_response_state(response: &mut Response, state: AssemblyResponseState) { + response.headers_mut().insert( + HEADER_X_TS_ASSEMBLY, + HeaderValue::from_static(state.as_str()), + ); +} fn body_as_reader( body: EdgeBody, @@ -201,11 +273,16 @@ fn restrict_accept_encoding(req: &mut Request) { // origin responds without compression. Adding encodings here would cause the // origin to compress its response even though the client never asked for it, // and the client would then receive content it cannot decode. + if !req.headers().contains_key(header::ACCEPT_ENCODING) { + return; + } let Some(current) = req .headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned) + .get_all(header::ACCEPT_ENCODING) + .iter() + .map(|value| value.to_str().ok()) + .collect::>>() + .map(|values| values.join(", ")) else { return; }; @@ -273,13 +350,167 @@ fn accept_encoding_qvalue(header_value: &str, target: &str) -> Option { matched_qvalue } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReaderEncodingError { + Malformed, + NoAcceptableEncoding, +} + +fn parse_quality_value(value: &str) -> Option { + let value = value.trim(); + let (whole, fraction) = value + .split_once('.') + .map_or((value, None), |(whole, fraction)| (whole, Some(fraction))); + let fraction_is_valid = fraction.is_none_or(|fraction| { + fraction.len() <= 3 && fraction.bytes().all(|byte| byte.is_ascii_digit()) + }); + if !fraction_is_valid { + return None; + } + match whole { + "0" => value.parse().ok(), + "1" if fraction.is_none_or(|fraction| fraction.bytes().all(|byte| byte == b'0')) => { + Some(1.0) + } + _ => None, + } +} + +fn negotiate_reader_compression( + headers: &edgezero_core::http::HeaderMap, +) -> Result { + if !headers.contains_key(header::ACCEPT_ENCODING) { + return Ok(Compression::None); + } + + let mut qualities = Vec::<(String, f32)>::new(); + for field in headers.get_all(header::ACCEPT_ENCODING) { + let field = field.to_str().map_err(|_| ReaderEncodingError::Malformed)?; + for item in field + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + { + let mut parts = item.split(';'); + let token = parts + .next() + .map(str::trim) + .filter(|token| !token.is_empty()) + .ok_or(ReaderEncodingError::Malformed)? + .to_ascii_lowercase(); + if token != "*" && http::HeaderName::from_bytes(token.as_bytes()).is_err() { + return Err(ReaderEncodingError::Malformed); + } + let mut quality = 1.0; + let mut saw_quality = false; + for parameter in parts { + let (name, value) = parameter + .trim() + .split_once('=') + .ok_or(ReaderEncodingError::Malformed)?; + if !name.trim().eq_ignore_ascii_case("q") || saw_quality { + return Err(ReaderEncodingError::Malformed); + } + quality = parse_quality_value(value).ok_or(ReaderEncodingError::Malformed)?; + saw_quality = true; + } + if qualities.iter().any(|(seen, _)| seen == &token) { + return Err(ReaderEncodingError::Malformed); + } + qualities.push((token, quality)); + } + } + + let explicit = |name: &str| { + qualities + .iter() + .find_map(|(candidate, quality)| (candidate == name).then_some(*quality)) + }; + let wildcard = explicit("*"); + let quality_for = |name: &str| explicit(name).or(wildcard).unwrap_or(0.0); + // Identity is implicitly acceptable at q=1 unless explicitly excluded, or a + // wildcard q=0 excludes every unlisted coding. + let identity_quality = + explicit("identity").unwrap_or_else(|| if wildcard == Some(0.0) { 0.0 } else { 1.0 }); + + let candidates = [ + (Compression::Brotli, quality_for("br")), + (Compression::Gzip, quality_for("gzip")), + (Compression::Deflate, quality_for("deflate")), + (Compression::None, identity_quality), + ]; + let mut selected = None; + for (compression, quality) in candidates { + if quality > 0.0 && selected.is_none_or(|(_, best)| quality > best) { + selected = Some((compression, quality)); + } + } + selected + .map(|(compression, _)| compression) + .ok_or(ReaderEncodingError::NoAcceptableEncoding) +} + +fn set_response_compression(response: &mut Response, compression: Compression) { + let encoding = match compression { + Compression::None => None, + Compression::Gzip => Some("gzip"), + Compression::Deflate => Some("deflate"), + Compression::Brotli => Some("br"), + }; + if let Some(encoding) = encoding { + response + .headers_mut() + .insert(header::CONTENT_ENCODING, HeaderValue::from_static(encoding)); + } else { + response.headers_mut().remove(header::CONTENT_ENCODING); + } + let varies_on_encoding = response + .headers() + .get_all(header::VARY) + .iter() + .any(|value| { + value.to_str().is_ok_and(|value| { + value + .split(',') + .any(|name| name.trim().eq_ignore_ascii_case("accept-encoding")) + }) + }); + if !varies_on_encoding { + response + .headers_mut() + .append(header::VARY, HeaderValue::from_static("Accept-Encoding")); + } + response.headers_mut().remove(header::CONTENT_LENGTH); +} + +fn response_compression(response: &Response) -> Compression { + response + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .map(Compression::from_content_encoding) + .unwrap_or(Compression::None) +} + +fn encode_complete_body( + body: Vec, + compression: Compression, +) -> Result, Report> { + let mut encoder = BodyStreamEncoder::new(compression); + let mut encoded = encoder.encode_chunk(body)?; + encoded.extend_from_slice(&encoder.finish()?); + Ok(encoded) +} + /// Unified tsjs static serving: `/static/tsjs=` /// -/// Serves two types of bundles: +/// Serves three types of bundles: /// - **Unified bundle** (`tsjs-unified.min.js`): core + immediate (non-deferred) /// integration modules. /// - **Deferred module** (`tsjs-{id}.min.js`): a single self-contained IIFE for -/// modules loaded with `defer` (e.g., prebid). +/// modules loaded with `defer` (e.g., Prebid). +/// - **Standalone diagnostics module** (`tsjs-gpt_diagnostics.min.js`): delivered +/// only when the diagnostics integration is enabled and a document activates it. /// /// # Errors /// @@ -287,6 +518,7 @@ fn accept_encoding_qvalue(header_value: &str, target: &str) -> Option { pub fn handle_tsjs_dynamic( req: &Request, integration_registry: &IntegrationRegistry, + edge_header: EdgeCacheHeader, ) -> Result, Report> { const PREFIX: &str = "/static/tsjs="; const UNIFIED_FILENAMES: &[&str] = &["tsjs-unified.js", "tsjs-unified.min.js"]; @@ -301,10 +533,8 @@ pub fn handle_tsjs_dynamic( // Serve core + immediate modules (excludes deferred like prebid) let module_ids = integration_registry.js_module_ids_immediate(); let body = trusted_server_js::concatenate_modules(&module_ids); - let mut resp = serve_static_with_etag(&body, req, "application/javascript; charset=utf-8"); - resp.headers_mut() - .insert(HEADER_X_COMPRESS_HINT, HeaderValue::from_static("on")); - return Ok(resp); + let hash = trusted_server_js::concatenated_hash(&module_ids); + return Ok(serve_tsjs_static(req, &body, &hash, edge_header)); } if let Some(module_id) = parse_single_module_filename(filename) { @@ -312,24 +542,52 @@ pub fn handle_tsjs_dynamic( // are served as content-addressed standalone assets. Delivery remains // cookie-independent so the static response can stay publicly cached. let deferred_ids = integration_registry.js_module_ids_deferred(); - let diagnostics_standalone = module_id + let is_enabled_diagnostics_module = module_id == crate::integrations::gpt_diagnostics::GPT_DIAGNOSTICS_INTEGRATION_ID - && integration_registry.integration_enabled(module_id); - if !deferred_ids.contains(&module_id) && !diagnostics_standalone { + && integration_registry.is_enabled(module_id); + if !deferred_ids.contains(&module_id) && !is_enabled_diagnostics_module { return Ok(not_found_response()); } - if let Some(content) = trusted_server_js::module_bundle(module_id) { - let mut resp = - serve_static_with_etag(content, req, "application/javascript; charset=utf-8"); - resp.headers_mut() - .insert(HEADER_X_COMPRESS_HINT, HeaderValue::from_static("on")); - return Ok(resp); + if let (Some(content), Some(hash)) = ( + trusted_server_js::module_bundle(module_id), + trusted_server_js::single_module_hash(module_id), + ) { + return Ok(serve_tsjs_static(req, content, hash, edge_header)); } } Ok(not_found_response()) } +fn serve_tsjs_static( + req: &Request, + body: &str, + expected_hash: &str, + edge_header: EdgeCacheHeader, +) -> Response { + let mut response = serve_static_with_etag( + body, + req, + "application/javascript; charset=utf-8", + edge_header, + ); + if request_version_hash(req).is_some_and(|hash| hash == expected_hash) { + CachePolicy::public_immutable(Duration::from_secs(31_536_000)) + .apply_to_headers(response.headers_mut(), edge_header); + } + response + .headers_mut() + .insert(HEADER_X_COMPRESS_HINT, HeaderValue::from_static("on")); + response +} + +fn request_version_hash(req: &Request) -> Option<&str> { + req.uri().query()?.split('&').find_map(|pair| { + let (name, value) = pair.split_once('=')?; + (name == "v").then_some(value) + }) +} + /// Extract a module ID from a deferred-module filename like `tsjs-sourcepoint.min.js`. /// /// Returns `Some(&'static str)` if the filename matches a known JS module ID, @@ -361,6 +619,8 @@ struct ProcessResponseParams<'a> { suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, + /// See [`HtmlStreamProcessorParams::shared_template_authorized`]. + shared_template_authorized: bool, } struct PublisherBodyProcessor { @@ -384,9 +644,10 @@ impl PublisherBodyProcessor { settings, integration_registry, ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), - ad_bids_state: Arc::clone(¶ms.ad_bids_state), + ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), + shared_template_authorized: params.template_cache_key.is_some(), })?) } else if is_rsc_flight { Box::new(RscFlightUrlRewriter::new( @@ -428,6 +689,7 @@ fn process_response_streaming( body: EdgeBody, output: &mut W, params: &ProcessResponseParams, + output_compression: Compression, ) -> Result<(), Report> { let is_html = is_html_content_type(params.content_type); let is_rsc_flight = @@ -443,7 +705,7 @@ fn process_response_streaming( let compression = Compression::from_content_encoding(params.content_encoding); let config = PipelineConfig { input_compression: compression, - output_compression: compression, + output_compression, chunk_size: 8192, }; // Bound how much decoded gzip output may sit in the heap at once, using the @@ -465,6 +727,7 @@ fn process_response_streaming( ad_bids_state: params.ad_bids_state.clone(), suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.cloned(), + shared_template_authorized: params.shared_template_authorized, })?; StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) @@ -509,13 +772,21 @@ async fn process_response_streaming_async( params.content_encoding ); - let compression = Compression::from_content_encoding(¶ms.content_encoding); + let input_compression = Compression::from_content_encoding(¶ms.content_encoding); + // A C2 template is always identity bytes. Decode during the transform instead of + // recompressing and immediately decoding the entire buffered result afterwards. + let output_compression = if params.template_cache_key.is_some() { + Compression::None + } else { + input_compression + }; let mut processor = PublisherBodyProcessor::new(params, settings, integration_registry)?; process_body_chunks_async( body, output, &mut processor, - compression, + input_compression, + output_compression, settings.publisher.max_buffered_body_bytes, ) .await @@ -557,11 +828,12 @@ async fn process_body_chunks_async( body: EdgeBody, writer: &mut W, processor: &mut P, - compression: Compression, + input_compression: Compression, + output_compression: Compression, max_body_bytes: usize, ) -> Result<(), Report> { - let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); - let mut encoder = BodyStreamEncoder::new(compression); + let mut decoder = BodyStreamDecoder::new(input_compression, max_body_bytes); + let mut encoder = BodyStreamEncoder::new(output_compression); let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_body_bytes); while let Some(segments) = @@ -950,6 +1222,137 @@ struct HtmlStreamProcessorParams<'a> { ad_bids_state: Arc>>, suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option, + /// Whether a shared template was authorized for this response. + /// + /// Carried rather than re-derived so both seams see the same answer. See + /// [`effective_assembly_mode`]. + shared_template_authorized: bool, +} + +/// The diagnostics decision the template may carry. +/// +/// Diagnostics is request-scoped — activated by a cookie or query parameter, and +/// documented as an immutable per-request decision — so it must not reach a shared +/// template. +/// +/// It does not leak today even without this gate, but only by coincidence: +/// `requires_private_no_store()` is a strict superset of the conditions under which +/// a script is emitted, and that stamp lands before the C2 gate reads response +/// headers, so the gate refuses. Two independent conditions that happen to align, +/// with nothing enforcing the relationship. This makes the guarantee explicit; +/// `requires_private_no_store_is_a_superset_of_injection` keeps the coincidence as a +/// backstop if this gate is ever removed. +pub(crate) fn template_gpt_diagnostics( + mode: AssemblyMode, + decision: Option, +) -> Option { + match mode { + AssemblyMode::Inline => decision, + AssemblyMode::Esi => None, + } +} + +/// The marker emitted at the `` seam under [`AssemblyMode::Esi`], reserving the +/// place this reader's slots and bids are spliced into. +/// +/// An inert HTML comment, deliberately. Template schema v1 used an executable ESI include +/// tag here, when the `esi` crate resolved it at the edge. That crate was removed from the +/// render path because it truncates any element larger than its 16 KB chunk size, and +/// nothing has parsed ESI since. What remained was a tag that *looked* executable, would +/// have been executed by any ESI-enabled layer in front of us, and renders as text in a +/// browser if assembly is ever skipped. A comment cannot do any of those things: an +/// unassembled template degrades to a page with no ads rather than a page with a visible +/// tag. +/// +/// Carries no URL. Every byte here is a byte every reader of the shared template +/// receives, so nothing request-scoped may appear, and keeping a URL out also removes +/// any escaping question at the seam. +pub const AD_ASSEMBLY_SEAM: &str = ""; + +/// The mode the operator asked for, before availability is taken into account. +/// +/// Spelled once, because the mode has to mean the same thing at the cache key, at the +/// seam, and at both hit finalizers. Every one of those re-derived it from the same +/// `Option` chain, and the finalizers had no way to ask at all — which is why they +/// demanded a seam marker of a mode that emits none. +fn configured_assembly_mode(settings: &Settings) -> AssemblyMode { + settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::assembly_mode) + .unwrap_or_default() +} + +/// Whether this mode's `` seam emits [`AD_ASSEMBLY_SEAM`]. +/// +/// The property that decides whether a template is *expected* to have a hole in it, and +/// therefore whether the absence of one is a defect or the design. Only `Esi` splices per +/// reader. +/// +/// Matched exhaustively rather than compared against `Esi`, so a new mode has to state +/// its answer here instead of silently inheriting one. +fn mode_emits_seam_marker(mode: AssemblyMode) -> bool { + match mode { + AssemblyMode::Inline => false, + AssemblyMode::Esi => true, + } +} + +/// The assembly mode this response will actually be delivered under. +/// +/// The configured mode says what the operator wants; the cache key says whether it is +/// available. A shared mode with no key means the gate refused this response — the +/// origin set a cookie, declared a `Vary` the key does not cover, returned a non-200, +/// and so on — so there is no shared template to build and nothing downstream will +/// assemble one. +/// +/// When that happens the request falls back to [`AssemblyMode::Inline`] **entirely**, +/// at every seam. Falling back at one seam and not another is what produced the failure +/// this function exists to prevent: the `` seam emitted a legacy ESI tag because +/// the mode was `Esi`, while assembly was skipped because there was no key, so the reader +/// received a document with unresolved executable ESI markup in it and no bids at all. +/// +/// Bypassing is the *normal* case against a real origin, not an edge case, so this path +/// runs far more often than the shared one. +fn effective_assembly_mode(settings: &Settings, shared_template_authorized: bool) -> AssemblyMode { + let configured = configured_assembly_mode(settings); + if matches!(configured, AssemblyMode::Inline) || shared_template_authorized { + return configured; + } + log::debug!( + "assembly mode {configured:?} is unavailable for this response (no shared template \ + was authorized); falling back to inline" + ); + AssemblyMode::Inline +} + +/// What the `` seam should inject, given the assembly mode. +/// +/// Explicit rather than inferred. The previous shape read +/// `ad_slots_script.is_some()` inside the element handler, which silently coupled +/// two independent decisions: once [`template_ad_slots_script`] stopped emitting a +/// head script under a shared mode, body-close injection stopped with it. +/// +/// `Esi` emits [`AD_ASSEMBLY_SEAM`], an inert HTML comment marking where this reader's +/// slots and bids are spliced in. Assembly is a byte split on that comment, performed by +/// this crate on both the miss and the hit path; no ESI layer is involved. +pub(crate) fn body_close_injection( + mode: AssemblyMode, + head_script_present: bool, +) -> BodyCloseInjection { + match mode { + // Per-navigation and never shared, so gating on slot presence is correct. + AssemblyMode::Inline => { + if head_script_present { + BodyCloseInjection::InlineBids + } else { + BodyCloseInjection::None + } + } + // Constant across every request that reaches the transform — which is what + // makes it safe in a shared template. + AssemblyMode::Esi => BodyCloseInjection::Marker(AD_ASSEMBLY_SEAM.to_string()), + } } fn create_html_stream_processor( @@ -963,10 +1366,18 @@ fn create_html_stream_processor( params.origin_host, params.request_host, params.request_scheme, - ) - .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(params.gpt_diagnostics) - .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); + ); + + let assembly_mode = effective_assembly_mode(params.settings, params.shared_template_authorized); + let body_close = body_close_injection(assembly_mode, params.ad_slots_script.is_some()); + + let gpt_diagnostics = template_gpt_diagnostics(assembly_mode, params.gpt_diagnostics); + + let config = config + .with_ad_state(params.ad_slots_script, params.ad_bids_state) + .with_gpt_diagnostics(gpt_diagnostics) + .with_body_close(body_close) + .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); Ok(create_html_processor(config)) } @@ -1000,6 +1411,27 @@ pub enum PublisherResponse { /// Parameters for [`process_response_streaming`]. params: Box, }, + /// A shared template read from C2, to be assembled on the way out. + /// + /// Distinct from [`Self::Stream`] because the bytes are **already transformed** — + /// running them through `lol_html` again would inject a second tsjs `"#.to_string() + }))); + + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script, + ad_bids_state, + max_buffered_body_bytes: 16 * 1024 * 1024, + gpt_diagnostics, + body_close, + suppress_datadome_client_side_tag: false, + }; + + let mut processor = create_html_processor(config); + let out = processor + .process_chunk(DOCUMENT, true) + .expect("should process the document"); + String::from_utf8(out).expect("output should be utf8") + } + + fn every_shape() -> Vec { + let mut shapes = Vec::new(); + for ad_stack_ran in [false, true] { + for diagnostics_active in [false, true] { + for bids_available in [false, true] { + shapes.push(RequestShape { + ad_stack_ran, + diagnostics_active, + bids_available, + }); + } + } + } + shapes + } + + #[test] + fn shared_template_ad_seam_is_readable_and_versioned() { + let BodyCloseInjection::Marker(marker) = body_close_injection(AssemblyMode::Esi, false) + else { + panic!("ESI mode should emit a shared-template marker"); + }; + + assert_eq!( + (crate::platform::TEMPLATE_SCHEMA_VERSION, marker.as_str(),), + (4, ""), + "the readable seam and its cache schema must move together" + ); + } + + #[test] + fn shared_modes_render_byte_identical_documents_for_every_request_shape() { + let mode = AssemblyMode::Esi; + let shapes = every_shape(); + let baseline = render(mode, shapes[0]); + + for shape in &shapes[1..] { + let rendered = render(mode, *shape); + assert_eq!( + rendered, baseline, + "{mode:?}: rendered template differs for {shape:?}. A shared \ + template that varies by request freezes the first-filling \ + request's decision for every later reader." + ); + } + } + + #[test] + fn shared_mode_templates_contain_no_request_scoped_markers() { + // Byte-identity alone would be satisfied by rendering the same wrong + // thing every time, so also assert the specific things that must be + // absent. + let mode = AssemblyMode::Esi; + let rendered = render( + mode, + RequestShape { + ad_stack_ran: true, + diagnostics_active: true, + bids_available: true, + }, + ); + for forbidden in [ + ".adSlots", + ".bids=", + "__tsjs_gpt_diagnostics_active", + "history.replaceState", + ] { + assert!( + !rendered.contains(forbidden), + "{mode:?}: template contains request-scoped `{forbidden}`:\n{rendered}" + ); + } + } + + #[test] + fn inline_still_varies_by_request_as_it_must() { + // The shared-mode assertions would also pass if rendering were broken + // everywhere. Inline responses are per-navigation and never shared, so + // they *should* differ — this proves the test can tell the difference. + let with_ads = render( + AssemblyMode::Inline, + RequestShape { + ad_stack_ran: true, + diagnostics_active: false, + bids_available: true, + }, + ); + let without = render( + AssemblyMode::Inline, + RequestShape { + ad_stack_ran: false, + diagnostics_active: false, + bids_available: false, + }, + ); + assert_ne!( + with_ads, without, + "inline must still vary by request; if it does not, this harness is \ + not rendering what it claims to" + ); + assert!( + with_ads.contains(".adSlots"), + "inline with a matched slot should carry adSlots" + ); + } + } + + mod template_fingerprint_tests { + use super::*; + + /// Base settings with one integration's config replaced. + /// + /// Edits the parsed `[integrations]` map rather than appending TOML, so the two + /// fixtures differ in exactly the field under test — the base settings already + /// declare `[integrations.prebid]`, and a second table would not parse. + fn settings_with_prebid(enabled: bool, timeout_ms: u32) -> Settings { + let mut settings = create_test_settings(); + settings.integrations.insert( + "prebid".to_string(), + serde_json::json!({ + "enabled": enabled, + "server_url": "https://prebid.example.com/openrtb2/auction", + "external_bundle_url": "https://assets.example.com/prebid/bundle.js", + "timeout": timeout_ms, + }), + ); + settings + } + + #[test] + fn disabling_an_integration_changes_the_fingerprint() { + // The fingerprint was `concatenated_hash(all_module_ids())` — every module + // compiled into the binary, so a constant for that binary. Turning an + // integration off changed the injected `origin" + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ( + "content-security-policy", + "default-src 'self'; script-src 'nonce-reader-nonce'", + ), + ], + ); + } + + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "a response-bound CSP nonce and its HTML must never be reused from C2" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty() + ); + } + + #[tokio::test] + async fn a_post_is_never_answered_from_a_cached_get() { + // `handle_publisher_request` is the `*`-method fallback route, so a publisher + // path that renders a page on GET and accepts a form or webhook on POST reaches + // here for both. Serving the cached GET to the POST swallows the mutating + // request entirely: the origin never sees it, the caller gets 200 and a page, + // and nothing anywhere reports a problem. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + // Warm the cache with a GET. + let _ = run(&settings, &services, navigation_request()).await; + assert_eq!(stub.recorded_request_uris().len(), 1); + + let post = HttpRequest::builder() + .method(Method::POST) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .body(EdgeBody::from("field=value")) + .expect("should build post request"); + let _ = run(&settings, &services, post).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "the POST must reach the origin rather than being answered from the \ + cached GET" + ); + assert_eq!( + cache.entries.lock().expect("should lock entries").len(), + 1, + "and it must not store a template of its own" + ); + } + + #[tokio::test] + async fn an_authenticated_request_is_not_served_a_shared_template() { + // The stored template is perfectly cacheable; this request is not entitled + // to it. The store gate cannot express that, because it is a property of + // the reader rather than of the bytes — which is why the lookup re-checks. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + assert_eq!( + cache.entries.lock().expect("should lock entries").len(), + 1, + "the cold request should have populated the cache" + ); + + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let authenticated = HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .body(EdgeBody::empty()) + .expect("should build authenticated request"); + let _ = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + authenticated, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "an authenticated request must reach the origin rather than read a \ + shared template" + ); + } + } + + mod c2_gate_tests { + //! `cache::core` stores whatever it is handed and rejects nothing, so every + //! one of these conditions is the caller's to enforce. Each is a leak vector + //! or an eligibility rule, not a preference. + + use super::*; + use crate::creative_opportunities::AssemblyMode; + use edgezero_core::http::HeaderName; + + fn headers(pairs: &[(HeaderName, &str)]) -> edgezero_core::http::HeaderMap { + let mut map = edgezero_core::http::HeaderMap::new(); + for (name, value) in pairs { + map.insert( + name.clone(), + HeaderValue::from_str(value).expect("should build header value"), + ); + } + map + } + + fn shareable() -> edgezero_core::http::HeaderMap { + headers(&[(header::CACHE_CONTROL, "max-age=60")]) + } + + /// The shipped default: no operator has stated what the origin varies on, so the + /// key covers nothing. Responses without a `Vary` are unaffected; any `Vary` at + /// all disqualifies. + fn nothing_covered() -> VarySpec { + VarySpec::new([]) + } + + #[test] + fn an_unconfigured_deployment_never_caches_a_varying_response() { + // The fail-closed default. An operator who has not stated the origin's Vary + // must not acquire a shared cache by omission — and a real origin varies on + // something, so this is the common path, not an edge case. + // Deliberately not `Accept-Encoding`: the shared path normalizes supported + // content codings to one identity template, so that header is covered + // whatever the operator configured. Using it here would test the + // structural-coverage carve-out rather than the drift guard. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("rsc")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + ¬hing_covered(), + ), + Some(C2BypassReason::VaryNotCovered(VaryGap(vec![ + "rsc".to_string() + ]))), + "an unstated Vary must disqualify rather than silently under-key" + ); + } + + #[test] + fn an_origin_that_varies_on_cookie_is_refused_even_when_declared_independent() { + // The backstop that makes `origin_is_cookie_independent` safe to offer. The + // operator asserts their origin ignores cookies; if the origin then says + // otherwise, the assertion loses. Without this, a wrong assertion would + // silently cross-serve personalized HTML. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("Cookie")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + // The operator's assertion has already been applied here: this is + // `false` precisely because they declared independence. + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["cookie".to_string()]), + ), + Some(C2BypassReason::VaryCookie), + "the origin's declaration must override both cookie independence and an \ + accidentally configured per-cookie key" + ); + } + + #[test] + fn a_private_directive_on_a_second_cache_control_line_is_refused() { + // `HeaderMap::get` returns the first value only. An origin that sends + // `Cache-Control: public, max-age=300` and then `Cache-Control: private` on a + // separate line means exactly what one comma-joined line would mean, but the + // second line was invisible — so a response the origin marked private was + // written to a cache shared between readers. The `Vary` reads a few lines up + // already use `get_all` for the same reason. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("public, max-age=300"), + ); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("private")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "a directive on any Cache-Control line must disqualify the response" + ); + } + + #[test] + fn a_no_store_directive_on_a_second_cache_control_line_is_refused() { + // Same defect, the other directive that matters — `no-store` is the one an + // origin uses for a response that must not be written down anywhere. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable) + ); + } + + #[test] + fn cdn_specific_cache_policy_cannot_be_overridden_by_public_cache_control() { + for name in crate::response_privacy::CDN_CACHE_HEADERS { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("no-store"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "C2 must fail closed on the CDN-specific policy header {name}" + ); + } + } + + #[test] + fn unsupported_vendor_freshness_does_not_authorize_c2() { + for name in crate::response_privacy::CDN_CACHE_HEADERS + .iter() + .filter(|name| **name != "surrogate-control") + { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("max-age=60"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "the Fastly exception must not authorize the vendor policy {name}" + ); + } + } + + #[test] + fn observed_fastly_surrogate_policy_uses_edge_freshness_capped_by_configuration() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200, stale-while-revalidate=21600, stale-if-error=604800", + ), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), + ), + Ok(Duration::from_secs(300)), + "Fastly's edge freshness should take precedence over the shorter browser \ + lifetime, while the configured safety ceiling remains authoritative" + ); + } + + #[test] + fn fastly_surrogate_freshness_takes_precedence_over_standard_freshness() { + for (cache_control, surrogate_control, expected) in [ + ("public, max-age=300", "max-age=30", 30), + ("public, max-age=30", "max-age=300", 300), + ] { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, cache_control), + ( + header::HeaderName::from_static("surrogate-control"), + surrogate_control, + ), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), + ), + Ok(Duration::from_secs(expected)) + ); + } + } + + #[test] + fn surrogate_stale_windows_do_not_extend_fresh_reuse() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=300"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=20, stale-while-revalidate=600, stale-if-error=1200", + ), + (header::AGE, "10"), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), + ), + Ok(Duration::from_secs(10)), + "stale windows are validated metadata, not fresh C2 lifetime" + ); + } + + #[test] + fn ambiguous_or_unsupported_surrogate_policy_fails_closed() { + for (policy, expected) in [ + ("max-age", C2BypassReason::MalformedCachePolicy), + ( + "max-age=30, max-age=60", + C2BypassReason::MalformedCachePolicy, + ), + ("max-age=tomorrow", C2BypassReason::MalformedCachePolicy), + ("max-age=30, public", C2BypassReason::MalformedCachePolicy), + ("stale-if-error=60", C2BypassReason::NoPositiveFreshness), + ("max-age=0", C2BypassReason::NoPositiveFreshness), + ("max-age=30,", C2BypassReason::MalformedCachePolicy), + ] { + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(policy).expect("should build Surrogate-Control"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(expected), + "`{policy}` must fail closed" + ); + } + } + + #[test] + fn restrictive_surrogate_policy_is_never_overridden_by_standard_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(directive).expect("should build Surrogate-Control"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "`{directive}` must remain authoritative" + ); + } + } + + #[test] + fn restrictive_standard_policy_is_never_overridden_by_surrogate_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let publisher_headers = headers(&[ + ( + header::CACHE_CONTROL, + &format!("public, max-age=60, {directive}"), + ), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + ), + ]); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "standard `{directive}` must refuse C2 even with positive edge freshness" + ); + } + } + + #[test] + fn surrogate_control_can_authorize_fastly_edge_freshness_without_browser_freshness() { + let publisher_headers = headers(&[( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + )]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), + ), + Ok(Duration::from_secs(300)), + "Fastly edge freshness should not require browser freshness" + ); + } + + #[test] + fn repeated_cache_control_lines_without_a_disqualifier_still_cache() { + // The other direction: reading every value must not turn an ordinary + // multi-line `Cache-Control` into a bypass, or the fix would disable the + // cache instead of tightening it. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("public")); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + None + ); + } + + #[test] + fn origin_freshness_is_positive_age_adjusted_and_capped() { + let fresh_headers = headers(&[(header::CACHE_CONTROL, "public, max-age=300")]); + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &fresh_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), + ), + Ok(Duration::from_secs(60)) + ); + + let aged = headers(&[ + (header::CACHE_CONTROL, "s-maxage=50, max-age=300"), + (header::AGE, "35"), + ]); + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &aged, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), + ), + Ok(Duration::from_secs(15)) + ); + + let old_date_without_age = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ]); + let one_minute_later = httpdate::parse_http_date("Wed, 12 Aug 2026 08:01:00 GMT") + .expect("should parse fixture time"); + assert_eq!( + origin_shared_ttl_at( + &old_date_without_age, + one_minute_later, + Duration::from_secs(60), + ), + Err(C2BypassReason::NoPositiveFreshness), + "an old Date is apparent age even when an upstream omitted Age" + ); + } + + #[test] + fn zero_exhausted_missing_and_malformed_freshness_are_refused() { + for (map, expected) in [ + ( + headers(&[(header::CACHE_CONTROL, "max-age=0")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=60"), (header::AGE, "60")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "public")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=tomorrow")]), + C2BypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=\"60")]), + C2BypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=+60")]), + C2BypassReason::MalformedCachePolicy, + ), + ] { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(expected) + ); + } + } + + #[test] + fn expires_can_authorize_but_never_extend_an_expired_response() { + let now = httpdate::parse_http_date("Wed, 12 Aug 2026 08:00:00 GMT") + .expect("should parse fixture time"); + let fresh = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); + assert_eq!( + origin_shared_ttl_at(&fresh, now, Duration::from_secs(60)), + Ok(Duration::from_secs(30)) + ); + + let expired = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:01:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); + assert_eq!( + origin_shared_ttl_at(&expired, now, Duration::from_secs(60)), + Err(C2BypassReason::NoPositiveFreshness) + ); + } + + #[test] + fn request_semantics_bypass_c2_except_for_a_max_age_zero_reload() { + for (name, value) in [ + (header::CACHE_CONTROL, "no-cache"), + (header::CACHE_CONTROL, "max-age=30"), + (header::CACHE_CONTROL, "max-age=\"0"), + (header::CACHE_CONTROL, "min-fresh=10"), + (header::CACHE_CONTROL, "no-store"), + (header::PRAGMA, "no-cache"), + (header::PRAGMA, "legacy-extension, no-cache"), + (header::RANGE, "bytes=0-99"), + (header::IF_NONE_MATCH, "\"etag\""), + (header::IF_MODIFIED_SINCE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ] { + let map = headers(&[(name.clone(), value)]); + assert!(request_bypasses_c2(&map), "{name}: {value} must bypass"); + } + assert!( + !request_bypasses_c2(&headers(&[(header::CACHE_CONTROL, "max-age=0")])), + "a browser reload may reuse C2 because the assembled response and auction \ + are still rebuilt for this reader" + ); + assert!(!request_bypasses_c2(&headers(&[( + header::CACHE_CONTROL, + "public" + )]))); + } + + #[test] + fn a_wildcard_vary_is_refused() { + // `VarySpec::uncovered_by` filters `*` out, with a comment saying the + // eligibility gate handles it. It did not — nothing rejected the wildcard, so + // a response the origin said no key can select was shareable. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("*")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + ¬hing_covered(), + ), + Some(C2BypassReason::VaryWildcard) + ); + } + + #[test] + fn a_fully_covered_vary_is_cacheable() { + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, Accept-Encoding"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string(), "accept-encoding".to_string()]), + ), + None, + "a key covering everything the origin varies on is safe to store" + ); + } + + #[test] + fn config_drift_names_the_missing_header() { + // The failure this guards: the origin adds a header to its Vary, nobody + // updates config, and requests differing only in that header start sharing a + // template. The reason must name it, or diagnosing means a bisect. + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, next-router-prefetch"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(C2BypassReason::VaryNotCovered(VaryGap(vec![ + "next-router-prefetch".to_string() + ]))), + "the uncovered header must be named" + ); + } + + #[test] + fn a_vary_split_across_repeated_headers_is_still_checked() { + // Vary is a list header, so an origin may send it once or many times. Reading + // only the first would let the rest through unkeyed. + let mut varying = shareable(); + varying.append(header::VARY, HeaderValue::from_static("rsc")); + varying.append(header::VARY, HeaderValue::from_static("cookie")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(C2BypassReason::VaryCookie), + "a repeated Vary header must not hide names behind the first value" + ); + } + + #[test] + fn a_plain_shareable_html_200_is_cacheable() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + None, + "ESI shareable HTML 200 should be eligible" + ); + } + + #[test] + fn inline_mode_never_writes_a_template() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Inline, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::InlineMode), + "inline has no shared template to write" + ); + } + + #[test] + fn an_authorized_request_is_never_cached() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::AuthorizedRequest), + "an authenticated response must not enter a shared cache" + ); + } + + #[test] + fn a_forwarded_request_cookie_disqualifies_even_without_set_cookie() { + // The dangerous case: session established on an earlier request, so this + // response carries no Set-Cookie, has no Cache-Control at all, is a 200, + // and is HTML — yet is personalized because TS forwarded the Cookie to + // origin unchanged. Every other condition reports it cacheable. + let no_cache_control = edgezero_core::http::HeaderMap::new(); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + true, + StatusCode::OK, + "text/html", + &no_cache_control, + ¬hing_covered(), + ), + Some(C2BypassReason::CookieForwarded), + "cookie-personalized HTML must not become a shared template" + ); + } + + #[test] + fn an_origin_set_cookie_is_never_cached() { + let with_cookie = headers(&[ + (header::CACHE_CONTROL, "max-age=60"), + (header::SET_COOKIE, "sid=abc; Path=/"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &with_cookie, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginSetCookie), + "caching this would replay one visitor's cookie to the next" + ); + } + + #[test] + fn non_shareable_cache_control_is_refused_case_insensitively() { + for directive in [ + "private", + "no-store", + "no-cache", + "Private, max-age=60", + "NO-STORE", + "public, No-Cache", + ] { + let map = headers(&[(header::CACHE_CONTROL, directive)]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "`{directive}` should disqualify the response" + ); + } + } + + #[test] + fn a_datadome_block_is_refused_by_the_status_check() { + // DataDome replaces the document with a 403 + // (`integrations/datadome/protection.rs:778`). There is no separate + // marker to detect, and none is needed. + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::FORBIDDEN, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::NonOkStatus), + "a blocked document must not become the shared template" + ); + } + + #[test] + fn non_html_is_refused() { + for content_type in ["text/x-component", "application/json", ""] { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + content_type, + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::NotHtml), + "`{content_type}` has no HTML template to transform" + ); + } + } + + #[test] + fn unsupported_content_encoding_is_refused_before_representation_headers_change() { + let map = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::CONTENT_ENCODING, "zstd"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::UnsupportedContentEncoding) + ); + + let mut repeated = headers(&[(header::CACHE_CONTROL, "public, max-age=60")]); + repeated.append(header::CONTENT_TYPE, HeaderValue::from_static("text/html")); + repeated.append( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &repeated, + ¬hing_covered(), + ), + Some(C2BypassReason::MalformedRepresentationHeaders) + ); + } + + #[test] + fn leak_vectors_are_reported_before_mere_ineligibility() { + // A response that fails several conditions should name the most serious + // one, so an operator reading the log sees the security reason rather + // than a content-type quibble. + let map = headers(&[ + (header::CACHE_CONTROL, "private"), + (header::SET_COOKIE, "sid=abc"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::FORBIDDEN, + "application/json", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::AuthorizedRequest), + "authorization is the most serious disqualifier and should win" + ); + } + } + + mod template_neutrality_tests { + //! The gate for #1009's shared-template design. + //! + //! An "absence of per-user values" scan is not sufficient here: the bug + //! that nearly shipped was a *conditionally present* element whose own + //! content was per-URL. These tests assert byte-identity across requests + //! that differ only in the gating decision. + + use super::*; + use crate::creative_opportunities::{ + AssemblyMode, CreativeOpportunityFormat, CreativeOpportunitySlot, + }; + + pub(super) fn slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: "atf".to_string(), + gam_unit_path: Some("/99999/example/home".to_string()), + div_id: Some("ad-atf".to_string()), + page_patterns: vec!["/**".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }], + floor_price: None, + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + compiled_unit: None, + } + } + + pub(super) fn settings_with_slots() -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + // Construct the section rather than mutating it if present: the shared + // fixture does not carry `[creative_opportunities]`, and an `if let + // Some(..)` here would silently no-op and make the inline assertion + // below vacuous. + settings.creative_opportunities = Some(CreativeOpportunitiesConfig { + enabled: true, + gam_network_id: "99999".to_string(), + auction_timeout_ms: Some(500), + price_granularity: Default::default(), + section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, + section_segment: None, + slot: vec![slot()], + }); + settings + } + + #[test] + fn shared_modes_emit_no_head_script_regardless_of_the_gating_decision() { + let settings = settings_with_slots(); + let slots = [slot()]; + let mode = AssemblyMode::Esi; + let ran = template_ad_slots_script(mode, true, &settings, &slots, "/"); + let did_not_run = template_ad_slots_script(mode, false, &settings, &slots, "/"); + + assert_eq!( + ran, did_not_run, + "{mode:?}: the template must be byte-identical whether or not the ad \ + stack ran; a cached object cannot carry one request's consent, bot, \ + prefetch or kill-switch decision" + ); + assert_eq!( + ran, None, + "{mode:?}: adSlots belongs in the per-request seam, not the template" + ); + } + + #[test] + fn inline_mode_keeps_its_request_dependent_behaviour() { + // Inline responses are per-navigation and never shared, so gating is + // correct there. This guards against "fixing" the shared-mode bug by + // breaking the shipped path. + let settings = settings_with_slots(); + let slots = [slot()]; - let ec_context = EcContext::read_from_request(&settings, &req, &noop_services()) - .expect("should read EC context"); + assert!( + template_ad_slots_script(AssemblyMode::Inline, true, &settings, &slots, "/") + .is_some(), + "inline should emit adSlots when the ad stack runs" + ); + assert_eq!( + template_ad_slots_script(AssemblyMode::Inline, false, &settings, &slots, "/"), + None, + "inline should emit nothing when the ad stack does not run" + ); + } - assert_eq!( - ec_context.ec_value(), - Some(cookie_ec.as_str()), - "should resolve request EC ID from cookie" - ); - assert!( - ec_context.cookie_was_present(), - "should detect cookie was present" - ); - assert_eq!( - ec_context.existing_cookie_ec_id(), - Some(cookie_ec.as_str()), - "should return cookie EC value for revocation" - ); - } + #[test] + fn shared_modes_are_neutral_across_differing_slot_matches() { + // Slot matching folds in the request path. Under a shared mode even + // that must not reach the template. + let settings = settings_with_slots(); - /// Drive `handle_publisher_request` with no creative opportunities — a plain - /// proxy with no server-side auction. Hides the auction/EC wiring so callers - /// read like a simple `(settings, services, req)` proxy. - async fn run_publisher_proxy( - settings: &Settings, - services: &RuntimeServices, - req: Request, - ) -> PublisherResponse { - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let mut ec_context = - EcContext::read_from_request(settings, &req, services).expect("should read EC context"); - handle_publisher_request( - settings, - services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[], - registry: None, - }, - req, - ) - .await - .expect("should proxy publisher request") + let matched = template_ad_slots_script( + AssemblyMode::Esi, + true, + &settings, + &[slot()], + "/news/article", + ); + let unmatched = + template_ad_slots_script(AssemblyMode::Esi, true, &settings, &[], "/other"); + + assert_eq!( + matched, unmatched, + "the template must not vary with slot matching under a shared mode" + ); + } } mod ssat_cache_policy_tests { @@ -4849,6 +12225,15 @@ mod tests { .expect("should parse settings with auction and creative opportunities enabled") } + fn settings_with_disabled_ad_templates() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n\ + [creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled ad templates") + } + fn settings_with_dispatching_provider() -> Settings { let toml = format!( "{}\n[auction]\nenabled = true\nproviders = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n\ @@ -4907,13 +12292,16 @@ mod tests { .expect("should build conditional navigation request") } - fn queue_cacheable_html_response(stub: &StubHttpClient) { + fn queue_html_response_with_cache_control( + stub: &StubHttpClient, + cache_control: &'static str, + ) { stub.push_response_with_headers( 200, b"origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), + ("cache-control", cache_control), ("etag", ORIGIN_ETAG), ("last-modified", ORIGIN_LAST_MODIFIED), ("surrogate-control", "max-age=300"), @@ -4958,6 +12346,7 @@ mod tests { registry: None, }, req, + EdgeCacheHeader::SMaxageFallback, ) .await .expect("should proxy publisher request") @@ -4967,6 +12356,7 @@ mod tests { match response { PublisherResponse::Buffered(response) | PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } | PublisherResponse::PassThrough { response, .. } => response.into_parts().0, } } @@ -4983,7 +12373,7 @@ mod tests { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_cacheable_html_response(&stub); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -5076,11 +12466,11 @@ mod tests { } #[tokio::test] - async fn navigation_without_matched_slots_preserves_origin_cache_policy() { + async fn navigation_without_matched_slots_uses_short_browser_cache_policy() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_cacheable_html_response(&stub); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -5126,7 +12516,7 @@ mod tests { ); for (header_name, expected) in [ - (header::CACHE_CONTROL, "public, max-age=300"), + (header::CACHE_CONTROL, "max-age=60"), (header::ETAG, ORIGIN_ETAG), (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), ( @@ -5157,6 +12547,102 @@ mod tests { } } + #[tokio::test] + async fn disabled_ad_templates_use_short_browser_cache_policy() { + // Arrange + let settings = settings_with_disabled_ad_templates(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let response = run_with_slots( + &settings, + &services, + &slots, + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "disabled server-side ad templates should not bypass the origin cache" + ); + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("max-age=60"), + "disabled server-side ad templates should use the short browser cache policy" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "disabled server-side ad templates should preserve {header_name}" + ); + } + } + + #[tokio::test] + async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "origin {cache_control} policy should not be weakened" + ); + } + } + #[tokio::test] async fn eligible_navigation_rejects_unexpected_origin_304() { for content_type in [None, Some("text/html; charset=utf-8")] { @@ -5200,7 +12686,9 @@ mod tests { // Assert let response = match response { PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { panic!("unexpected origin 304 should return a buffered response") } }; @@ -5283,7 +12771,9 @@ mod tests { // Assert let response = match response { PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { panic!("noneligible origin 304 should remain buffered") } }; @@ -5357,7 +12847,8 @@ mod tests { *response.body_mut() = body; response } - PublisherResponse::Stream { response, .. } => response, + PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } => response, }; assert_eq!(response.status(), StatusCode::OK); @@ -5619,6 +13110,7 @@ mod tests { registry: None, }, req, + EdgeCacheHeader::SMaxageFallback, ) .await .expect("should proxy publisher request"); @@ -5984,39 +13476,69 @@ mod tests { #[test] fn server_side_ad_stack_runs_only_when_all_auction_gates_pass() { + let enabled_config = ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: true, + }; assert!( - should_run_server_side_ad_stack(true, true, false, false, true, true, true), - "GET, real navigation, matched slots, and consent should run TS ad stack" + should_run_server_side_ad_stack(true, true, false, false, true, true, enabled_config,), + "GET, real navigation, enabled templates, matched slots, and consent should run TS ad stack" ); assert!( - !should_run_server_side_ad_stack(false, true, false, false, true, true, true), + !should_run_server_side_ad_stack(false, true, false, false, true, true, enabled_config,), "non-GET requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, false, false, false, true, true, true), + !should_run_server_side_ad_stack(true, false, false, false, true, true, enabled_config,), "non-document requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, true, false, true, true, true), + !should_run_server_side_ad_stack(true, true, true, false, true, true, enabled_config,), "prefetch requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, true, true, true, true), + !should_run_server_side_ad_stack(true, true, false, true, true, true, enabled_config,), "bot requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, false, true, true), + !should_run_server_side_ad_stack(true, true, false, false, false, true, enabled_config,), "requests with no matching slots should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, false, true), + !should_run_server_side_ad_stack(true, true, false, false, true, false, enabled_config,), "requests without required consent should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, true, false), + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: false, + }, + ), "disabled [auction].enabled kill switch should skip TS ad stack and injection" ); + assert!( + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: false, + auction_enabled: true, + }, + ), + "disabled [creative_opportunities].enabled switch should skip TS ad stack and injection" + ); } #[tokio::test] @@ -6039,7 +13561,7 @@ mod tests { read_count: Arc::clone(&read_count), body_close_processed_at: Arc::clone(&body_close_processed_at), }; - let ad_bids_state = Arc::new(Mutex::new(None)); + let ad_bids_state = AdBidsState::default(); let ctx = AuctionCollectCtx { dispatched, telemetry: AuctionTelemetryCarry { @@ -6084,7 +13606,7 @@ mod tests { let settings = create_test_settings(); let services = noop_services(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let ad_bids_state = Arc::new(Mutex::new(None)); + let ad_bids_state = AdBidsState::default(); let mut state = AuctionHoldState::new( DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( test_auction_request(), @@ -6133,6 +13655,7 @@ mod tests { ); assert!( ad_bids_state + .script_cell() .lock() .expect("should lock bid state") .is_none(), @@ -6150,6 +13673,7 @@ mod tests { ); assert!( ad_bids_state + .script_cell() .lock() .expect("should lock bid state") .is_some(), @@ -6581,6 +14105,59 @@ mod tests { ); } + #[test] + fn esi_reader_encoding_negotiation_honours_quality_identity_and_repeated_fields() { + let headers = |values: &[&str]| { + let mut headers = edgezero_core::http::HeaderMap::new(); + for value in values { + headers.append( + header::ACCEPT_ENCODING, + HeaderValue::from_str(value).expect("should build accept-encoding"), + ); + } + headers + }; + + assert_eq!( + negotiate_reader_compression(&headers(&[])), + Ok(Compression::None) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.8", "br;q=0.4, identity;q=0.1"])), + Ok(Compression::Gzip) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip, br"])), + Ok(Compression::Brotli), + "server preference breaks an equal-quality tie" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.5"])), + Ok(Compression::None), + "implicit identity has q=1" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["zstd, identity;q=0"])), + Err(ReaderEncodingError::NoAcceptableEncoding) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=invalid"])), + Err(ReaderEncodingError::Malformed) + ); + for malformed in [ + "gzip;q=1e-1", + "gzip;q=0.1234", + "gzip;q=1.001", + "not a coding;q=1", + ] { + assert_eq!( + negotiate_reader_compression(&headers(&[malformed])), + Err(ReaderEncodingError::Malformed), + "{malformed} is not valid Accept-Encoding syntax" + ); + } + } + #[test] fn tsjs_dynamic_returns_not_found_for_unknown_filename() { let settings = create_test_settings(); @@ -6591,7 +14168,8 @@ mod tests { "https://publisher.example/static/tsjs=unknown.js", ); - let response = handle_tsjs_dynamic(&req, ®istry).expect("should handle tsjs request"); + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); assert_eq!(response.status(), StatusCode::NOT_FOUND); } @@ -6605,7 +14183,8 @@ mod tests { "https://publisher.example/static/tsjs=tsjs-unified.min.js", ); - let response = handle_tsjs_dynamic(&req, ®istry).expect("should handle tsjs request"); + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); assert_eq!(response.status(), StatusCode::OK); } @@ -6627,7 +14206,8 @@ mod tests { HeaderValue::from_static("__Host-ts-console=1"), ); - let response = handle_tsjs_dynamic(&req, ®istry).expect("should handle tsjs request"); + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); assert_eq!(response.status(), StatusCode::OK); assert!(!response.headers().contains_key(header::SET_COOKIE)); @@ -6689,7 +14269,8 @@ mod tests { "https://publisher.example/static/tsjs=tsjs-prebid.min.js", ); - let response = handle_tsjs_dynamic(&req, ®istry).expect("should handle tsjs request"); + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); assert_eq!( response.status(), StatusCode::OK, @@ -6718,7 +14299,8 @@ mod tests { "https://publisher.example/static/tsjs=tsjs-prebid.min.js", ); - let response = handle_tsjs_dynamic(&req, ®istry).expect("should handle tsjs request"); + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); assert_eq!( response.status(), StatusCode::NOT_FOUND, @@ -6736,7 +14318,8 @@ mod tests { "https://publisher.example/static/tsjs=tsjs-evil.min.js", ); - let response = handle_tsjs_dynamic(&req, ®istry).expect("should handle tsjs request"); + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); assert_eq!( response.status(), StatusCode::NOT_FOUND, @@ -6799,6 +14382,9 @@ mod tests { let body = EdgeBody::from(compressed); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -6806,7 +14392,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -6848,6 +14434,9 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -6855,7 +14444,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -6886,6 +14475,9 @@ mod tests { let registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -6893,7 +14485,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7002,6 +14594,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7009,7 +14604,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7056,6 +14651,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7063,7 +14661,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7113,6 +14711,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "deflate".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7120,7 +14721,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7170,6 +14771,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "br".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7177,7 +14781,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7227,6 +14831,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "br".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7234,7 +14841,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7272,6 +14879,9 @@ mod tests { fn non_html_stream_params(content_encoding: &str) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7279,7 +14889,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7459,8 +15069,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let state = Arc::new(Mutex::new(None)); + let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7524,8 +15137,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let state = Arc::new(Mutex::new(None)); + let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7593,6 +15209,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7600,7 +15219,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: Some(test_auction_request()), dispatched_auction: Some(DispatchedAuction::empty_for_test( @@ -7653,6 +15272,9 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7660,7 +15282,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7754,7 +15376,15 @@ mod tests { } fn streaming_finalize_response(params: OwnedProcessResponseParams, body: EdgeBody) -> EdgeBody { - let settings = Arc::new(create_test_settings()); + streaming_finalize_response_with_settings(params, body, create_test_settings()) + } + + fn streaming_finalize_response_with_settings( + params: OwnedProcessResponseParams, + body: EdgeBody, + settings: Settings, + ) -> EdgeBody { + let settings = Arc::new(settings); let registry = Arc::new( IntegrationRegistry::new(&settings).expect("should create integration registry"), ); @@ -7787,6 +15417,9 @@ mod tests { dispatched_auction: Option, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7797,7 +15430,7 @@ mod tests { r#""# .to_string(), ), - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: dispatched_auction.as_ref().map(|_| test_auction_request()), dispatched_auction, @@ -7807,6 +15440,40 @@ mod tests { } } + #[test] + fn streaming_finalize_emits_gam_attribution_head_before_origin_eof() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "gpt", + &serde_json::json!({ + "enabled": true, + "gam_attribution_enabled": true + }), + ) + .expect("should insert GPT config"); + + let body = streaming_finalize_response_with_settings( + html_stream_params("", None), + origin_chunk_then_pending(bytes::Bytes::from_static( + b"

origin remains pending

", + )), + settings, + ); + let html = String::from_utf8(first_lazy_body_chunk(body).to_vec()) + .expect("should emit UTF-8 HTML"); + + assert!( + html.contains("__tsjs_gam_attribution_enabled=true"), + "first rewritten head chunk should carry the primary activation flag: {html}" + ); + assert!( + html.contains("data-ts-gam-attribution=\"true\""), + "first rewritten head chunk should authorize the bundle fallback: {html}" + ); + } + #[test] fn streaming_finalize_emits_compressed_html_before_origin_eof() { // The FCP regression from #849: the lazy body must emit its first @@ -8133,6 +15800,9 @@ mod tests { let ec_context = EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8140,7 +15810,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: Some(AuctionObservationContext::from_parts( AuctionSource::SpaNavigation, "proxy.example.com", @@ -8316,6 +15986,9 @@ mod tests { .map(bytes::Bytes::copy_from_slice) .collect(); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8326,7 +15999,7 @@ mod tests { r#""# .to_string(), ), - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: Some(test_auction_request()), dispatched_auction: Some(DispatchedAuction::empty_for_test( @@ -8385,8 +16058,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let bids_script = r#""#; - let state = Arc::new(Mutex::new(Some(bids_script.to_string()))); + let state = AdBidsState::with_script(bids_script); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8441,6 +16117,9 @@ mod tests { // Claim gzip encoding but feed non-gzip bytes. The GzDecoder will // error as soon as it tries to read the gzip header. let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8448,7 +16127,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8550,6 +16229,9 @@ mod tests { let body = EdgeBody::from(html.to_vec()); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8557,7 +16239,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8608,6 +16290,9 @@ mod tests { // Small, single-fragment RSC script — placeholder path (not fallback). let html = br#""#; let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8615,7 +16300,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8652,8 +16337,9 @@ mod tests { #[cfg(test)] mod creative_opportunities_tests { use super::super::{ - MatchedSlotsContext, build_ad_slots_script, build_auction_request, build_bid_map, - build_bids_script, diagnostics_auction_id, html_escape_for_script, write_bids_to_state, + AdBidsState, MatchedSlotsContext, build_ad_slots_script, build_auction_request, + build_bid_map, build_bids_script, diagnostics_auction_id, html_escape_for_script, + write_bids_to_state, }; use crate::auction::types::{ApsRendererV1, ApsTagType, Bid, BidRenderer, MediaType}; use crate::consent::ConsentContext; @@ -8674,10 +16360,15 @@ mod tests { fn make_config() -> CreativeOpportunitiesConfig { CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "21765378893".to_string(), auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, section_segment: None, slot: Vec::new(), } @@ -8736,9 +16427,14 @@ mod tests { #[test] fn ad_slots_script_contains_slot_data() { - let slots = vec![make_slot()]; + let mut slot = make_slot(); + slot.targeting + .insert("ts".to_string(), "operator-value".to_string()); + let slots = vec![slot]; let config = make_config(); let script = build_ad_slots_script(&slots, &config, "/"); + let slot_json = crate::publisher::build_slot_json(&slots[0], &config, "example") + .expect("should build slot JSON"); assert!( script.contains("window.tsjs=window.tsjs||{}"), "should initialise tsjs namespace" @@ -8753,6 +16449,10 @@ mod tests { !script.contains("__ts_request_id"), "must NOT contain request_id" ); + assert_eq!( + slot_json["targeting"]["ts"], "operator-value", + "should forward operator-provided ts targeting verbatim" + ); } #[test] @@ -8963,7 +16663,7 @@ mod tests { ), ); - let state = std::sync::Arc::new(std::sync::Mutex::new(None)); + let state = AdBidsState::default(); write_bids_to_state( &winning_bids, PriceGranularity::Dense, @@ -8974,6 +16674,7 @@ mod tests { Some(&auction_request.id), ); let script = state + .script_cell() .lock() .expect("should lock initial bid state") .clone() @@ -9008,6 +16709,7 @@ mod tests { Some(&auction_request.id), ); let empty_script = state + .script_cell() .lock() .expect("should lock empty initial bid state") .clone() @@ -10353,6 +18055,14 @@ mod tests { Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } + fn settings_with_co_templates_disabled() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled templates") + } + async fn run_page_bids( settings: &Settings, orchestrator: &AuctionOrchestrator, @@ -11102,6 +18812,35 @@ mod tests { ); } + #[tokio::test] + async fn disabled_server_side_ad_templates_return_no_slots_or_bids() { + // The dedicated template switch must suppress publisher/page-bids + // delivery without using the global auction switch. + let settings = settings_with_co_templates_disabled(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let req = make_page_bids_request("/2024/01/my-article/"); + + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 0, + "disabled server-side ad templates must not return slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "disabled server-side ad templates must not produce bids" + ); + } + #[tokio::test] async fn consent_denied_returns_no_slots_or_bids() { // When consent denies the server-side auction (here: Jurisdiction @@ -11453,6 +19192,7 @@ mod tests { registry: None, }, req, + EdgeCacheHeader::SMaxageFallback, ) .await .expect("should proxy publisher request"); @@ -11536,6 +19276,7 @@ mod tests { registry: None, }, req, + EdgeCacheHeader::SMaxageFallback, ) .await .expect("should proxy publisher request"); diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 21ba9f20b..b926273fb 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -9,8 +9,12 @@ //! cache such as Cloudflare would otherwise serve an operator/origin //! `Cache-Control: public` on a cookie-bearing response as-is. -use edgezero_core::http::{HeaderName, HeaderValue, Response, header}; +use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, Response, header}; +use crate::cache_policy::{ + cache_control_headers_are_private_or_no_store, is_edge_cache_header_name, + remove_edge_cache_headers, +}; use crate::settings::Settings; /// CDN-targeted cache headers stripped from every cookie-bearing response. @@ -30,21 +34,66 @@ fn strip_cdn_cache_headers(response: &mut Response) { } } -/// Forces synthesized HTML to be private and non-storable. +/// Whether `Cache-Control` already forbids shared caching. /// -/// Use this exact policy whenever Trusted Server changes an origin HTML -/// representation with request-specific content: force `private, no-store`, -/// remove origin validators, and remove all CDN-targeted cache directives. -pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { +/// Extracted because both arms of the cookie-privacy net below need it. +/// +/// `publisher::c2_bypass_reason` deliberately does **not** call this and keeps its own +/// copy: it additionally treats `no-cache` as non-shareable, because "revalidate before +/// reuse" is correct for an HTTP cache and too permissive for a spike-owned one. The +/// duplicate is the stricter of the two, so consolidating them would loosen the shared- +/// template gate rather than tidy it. +/// +/// Directives are case-insensitive (RFC 9111 §5.2), so `No-Store` and `Private` +/// count. `no-cache` deliberately does **not**: it requires revalidation before +/// reuse, not a refusal to store, so a `no-cache` response is still shareable. +/// Callers needing the stricter reading must check it themselves. +#[must_use] +pub fn is_private_or_no_store(headers: &HeaderMap) -> bool { + cache_control_headers_are_private_or_no_store(headers) +} + +/// Reassert the terminal privacy invariant for a synthesized per-reader response. +/// +/// Call this after every configurable response mutation. It deliberately overwrites +/// `Cache-Control` and strips validators, expiry metadata, and CDN-specific cache +/// directives so a later integration cannot turn an assembled document into C3. +pub fn enforce_private_no_store(response: &mut Response) { response.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static("private, no-store"), ); - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); + for name in [ + header::ETAG.as_str(), + header::LAST_MODIFIED.as_str(), + header::EXPIRES.as_str(), + header::AGE.as_str(), + ] { + response.headers_mut().remove(name); + } strip_cdn_cache_headers(response); } +/// Forces synthesized HTML to be private and non-storable. +/// +/// Use this exact policy whenever Trusted Server changes an origin HTML +/// representation with request-specific content: force `private, no-store`, +/// remove origin validators, and remove all CDN-targeted cache directives. +pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { + enforce_private_no_store(response); +} + +/// Removes runtime edge-cache headers from a response finalized as uncacheable. +/// +/// Call this after any late response-header mutations so a final `private` or +/// `no-store` directive cannot coexist with an independently authoritative edge +/// cache header. +pub fn enforce_uncacheable_cache_privacy(response: &mut Response) { + if is_private_or_no_store(response.headers()) { + remove_edge_cache_headers(response.headers_mut()); + } +} + /// Forces cookie-bearing responses to stay private to shared caches. /// /// Any response that sets a per-user cookie (notably the EC identity cookie) @@ -63,14 +112,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. strip_cdn_cache_headers(response); - // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match - // against a lowercased copy — `No-Store` / `Private` must count. - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let already_uncacheable = is_private_or_no_store(response.headers()); if !already_uncacheable { response.headers_mut().insert( header::CACHE_CONTROL, @@ -96,19 +138,13 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: &mut Response) { enforce_set_cookie_cache_privacy(response); - let response_is_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let response_is_uncacheable = is_private_or_no_store(response.headers()); + enforce_uncacheable_cache_privacy(response); for (key, value) in &settings.response_headers { if response_is_uncacheable && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) - || CDN_CACHE_HEADERS - .iter() - .any(|name| key.eq_ignore_ascii_case(name))) + || is_edge_cache_header_name(key)) { continue; } @@ -129,6 +165,8 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: response.headers_mut().insert(header_name, header_value); } + enforce_uncacheable_cache_privacy(response); + // Operator headers can themselves introduce Set-Cookie (alongside public // or surrogate cache headers) onto a previously cookieless response, which // the pre-apply pass could not see. Re-run the downgrade so the final @@ -296,6 +334,40 @@ mod tests { } } + #[test] + fn terminal_private_stamp_removes_every_cache_and_validator_header() { + let mut response = response_builder() + .header(header::CACHE_CONTROL, "public, s-maxage=600") + .header(header::ETAG, "\"origin\"") + .header(header::LAST_MODIFIED, "Wed, 12 Aug 2026 00:00:00 GMT") + .header(header::EXPIRES, "Wed, 12 Aug 2026 01:00:00 GMT") + .header(header::AGE, "30") + .header("surrogate-control", "max-age=600") + .header("cdn-cache-control", "public, max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + enforce_private_no_store(&mut response); + + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store" + ); + for name in [ + header::ETAG.as_str(), + header::LAST_MODIFIED.as_str(), + header::EXPIRES.as_str(), + header::AGE.as_str(), + "surrogate-control", + "cdn-cache-control", + ] { + assert!( + !response.headers().contains_key(name), + "terminal private stamp must strip {name}" + ); + } + } + #[test] fn applies_operator_headers_on_cookieless_response() { let settings = settings_with_response_headers(&[("x-operator", "value")]); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 598c00056..739a5ee43 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1,6 +1,7 @@ #[cfg(test)] use config::{Config, Environment, File, FileFormat}; use error_stack::{Report, ResultExt}; +use glob::{MatchOptions, Pattern}; use regex::Regex; use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; @@ -8,10 +9,12 @@ use std::collections::{HashMap, HashSet}; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; +use std::time::Duration; use url::Url; use validator::{Validate, ValidationError}; use crate::auction_config_types::AuctionConfig; +use crate::cache_policy::{CachePolicy, CacheVisibility}; use crate::consent_config::ConsentConfig; use crate::creative_opportunities::CreativeOpportunitiesConfig; use crate::error::TrustedServerError; @@ -1866,6 +1869,476 @@ fn validate_tinybird_secret(value: &str, setting: &str) -> Result<(), Report, +} + +impl CacheSettings { + fn normalize(&mut self) { + for rule in &mut self.asset_rules { + rule.normalize(); + } + } + + /// Eagerly validate runtime-only cache settings artifacts. + /// + /// # Errors + /// + /// Returns a configuration error if any rule ID is duplicate, or if an + /// enabled rule has an invalid policy/matcher or cannot compile its regex/glob. + pub fn prepare_runtime(&self) -> Result<(), Report> { + let mut seen_ids = HashSet::new(); + for rule in &self.asset_rules { + if rule.id.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: "cache.asset_rules id must not be empty".to_string(), + })); + } + if !seen_ids.insert(rule.id.clone()) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("cache.asset_rules contains duplicate id `{}`", rule.id), + })); + } + } + for rule in &self.asset_rules { + rule.prepare_runtime()?; + } + Ok(()) + } + + /// Resolve the first enabled asset cache rule that matches `path`. + /// + /// # Errors + /// + /// Returns a configuration error if a lazily prepared matcher unexpectedly + /// fails to compile. + pub fn asset_policy_for_path( + &self, + path: &str, + ) -> Result, Report> { + for rule in &self.asset_rules { + if rule.matches_path(path)? { + return Ok(Some(rule.cache_policy())); + } + } + Ok(None) + } +} + +/// A configurable cache rule for publisher-origin or rehosted static assets. +#[derive(Debug, Default, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CacheAssetRule { + /// Stable operator-facing identifier for logs/tests/config errors. + pub id: String, + /// Whether this rule participates in matching. + #[serde(default)] + pub enabled: bool, + /// Built-in framework/static preset matcher. + #[serde(default)] + pub preset: Option, + /// Raw path prefix matcher. + #[serde(default)] + pub path_prefix: Option, + /// Single glob matcher retained for concise configs. + #[serde(default)] + pub path_glob: Option, + /// Multiple glob matchers. + #[serde(default)] + pub path_globs: Vec, + /// Regex matcher applied to the request path. + #[serde(default)] + pub path_regex: Option, + /// File extensions matched against the request path, case-insensitively. + #[serde(default)] + pub extensions: Vec, + /// Bundler fingerprint style required in the filename before matching. + #[serde(default)] + pub fingerprint_style: Option, + /// Browser-facing cache visibility. + #[serde(default)] + pub visibility: CachePolicyVisibility, + /// Browser cache TTL rendered as `max-age`. + #[serde(default)] + pub browser_ttl_seconds: Option, + /// Shared edge cache TTL rendered as runtime-specific edge control. + #[serde(default)] + pub edge_ttl_seconds: Option, + /// Optional stale-while-revalidate duration. + #[serde(default)] + pub stale_while_revalidate_seconds: Option, + /// Optional stale-if-error duration. + #[serde(default)] + pub stale_if_error_seconds: Option, + /// Whether browser caches may treat the response as immutable. + #[serde(default)] + pub immutable: bool, + #[serde(skip)] + compiled_regex: OnceLock>, + #[serde(skip)] + compiled_globs: OnceLock, String>>, +} + +impl CacheAssetRule { + fn normalize(&mut self) { + self.id = self.id.trim().to_string(); + self.path_prefix = self + .path_prefix + .take() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + self.path_glob = self + .path_glob + .take() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + self.path_globs = self + .path_globs + .iter() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .collect(); + self.path_regex = self + .path_regex + .take() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + self.extensions = self + .extensions + .iter() + .map(|value| value.trim().trim_start_matches('.').to_ascii_lowercase()) + .filter(|value| !value.is_empty()) + .collect(); + } + + fn prepare_runtime(&self) -> Result<(), Report> { + if !self.enabled { + return Ok(()); + } + + self.validate_matcher_shape()?; + self.compiled_regex().map(|_| ())?; + self.compiled_globs().map(|_| ())?; + self.validate_policy_shape()?; + Ok(()) + } + + fn validate_matcher_shape(&self) -> Result<(), Report> { + if self.path_glob.is_some() && !self.path_globs.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` must use path_glob or path_globs, not both", + self.id + ), + })); + } + + let matcher_count = usize::from(self.preset.is_some()) + + usize::from(self.path_prefix.is_some()) + + usize::from(self.path_glob.is_some() || !self.path_globs.is_empty()) + + usize::from(self.path_regex.is_some()) + + usize::from(!self.extensions.is_empty()); + + if matcher_count != 1 { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` must configure exactly one matcher", + self.id + ), + })); + } + Ok(()) + } + + fn validate_policy_shape(&self) -> Result<(), Report> { + if self.visibility == CachePolicyVisibility::Private { + if self.edge_ttl_seconds.is_some() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` sets edge_ttl_seconds with private visibility; private rules must use browser_ttl_seconds", + self.id + ), + })); + } + if self.browser_ttl_seconds.is_none() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` with private visibility must configure browser_ttl_seconds", + self.id + ), + })); + } + } else if self.browser_ttl_seconds.is_none() && self.edge_ttl_seconds.is_none() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` must configure browser_ttl_seconds or edge_ttl_seconds", + self.id + ), + })); + } + + if !self.immutable { + return Ok(()); + } + + if self + .browser_ttl_seconds + .is_none_or(|browser_ttl| browser_ttl == 0) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` sets immutable without a positive browser_ttl_seconds", + self.id + ), + })); + } + + let preset_is_content_addressed = + matches!(self.preset, Some(CacheAssetPreset::NextJsStatic)); + if !preset_is_content_addressed && self.fingerprint_style.is_none() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` sets immutable without fingerprint_style or a content-addressed preset", + self.id + ), + })); + } + + Ok(()) + } + + fn compiled_regex(&self) -> Result, Report> { + let Some(pattern) = self.path_regex.as_deref() else { + return Ok(None); + }; + match self + .compiled_regex + .get_or_init(|| Regex::new(pattern).map_err(|err| err.to_string())) + { + Ok(regex) => Ok(Some(regex)), + Err(message) => Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` path_regex `{pattern}` failed to compile: {message}", + self.id + ), + })), + } + } + + fn compiled_globs(&self) -> Result, Report> { + if self.path_glob.is_none() && self.path_globs.is_empty() { + return Ok(None); + } + + match self.compiled_globs.get_or_init(|| { + let mut compiled = Vec::new(); + let source_patterns = self + .path_glob + .iter() + .chain(self.path_globs.iter()) + .map(String::as_str); + for pattern in source_patterns { + compile_cache_asset_glob_patterns(pattern, &mut compiled)?; + } + Ok(compiled) + }) { + Ok(patterns) => Ok(Some(patterns.as_slice())), + Err(message) => Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cache.asset_rules `{}` glob matcher failed to compile: {message}", + self.id + ), + })), + } + } + + fn matches_path(&self, path: &str) -> Result> { + if !self.enabled || !self.matcher_matches_path(path)? { + return Ok(false); + } + + if let Some(style) = self.fingerprint_style + && !filename_contains_fingerprint(path, style) + { + log::debug!( + "cache asset rule `{}` rejects path `{path}` because the filename has no {style:?} fingerprint", + self.id + ); + return Ok(false); + } + + Ok(true) + } + + fn matcher_matches_path(&self, path: &str) -> Result> { + if let Some(preset) = self.preset { + return Ok(preset.matches_path(path)); + } + if let Some(prefix) = self.path_prefix.as_deref() { + return Ok(path.starts_with(prefix)); + } + if let Some(patterns) = self.compiled_globs()? { + return Ok(patterns + .iter() + .any(|pattern| pattern.matches_with(path, CACHE_ASSET_GLOB_MATCH_OPTIONS))); + } + if let Some(regex) = self.compiled_regex()? { + return Ok(regex.is_match(path)); + } + if !self.extensions.is_empty() { + return Ok(path_extension(path).is_some_and(|extension| { + self.extensions + .iter() + .any(|candidate| candidate == &extension) + })); + } + Ok(false) + } + + fn cache_policy(&self) -> CachePolicy { + CachePolicy { + visibility: self.visibility.into(), + browser_ttl: self.browser_ttl_seconds.map(Duration::from_secs), + edge_ttl: self.edge_ttl_seconds.map(Duration::from_secs), + stale_while_revalidate: self.stale_while_revalidate_seconds.map(Duration::from_secs), + stale_if_error: self.stale_if_error_seconds.map(Duration::from_secs), + immutable: self.immutable, + } + } +} + +const CACHE_ASSET_GLOB_MATCH_OPTIONS: MatchOptions = MatchOptions { + case_sensitive: true, + require_literal_separator: true, + require_literal_leading_dot: false, +}; + +fn compile_cache_asset_glob_patterns( + pattern: &str, + compiled: &mut Vec, +) -> Result<(), String> { + compiled.push(Pattern::new(pattern).map_err(|err| err.to_string())?); + + if let Some(optional_recursive_start) = pattern.find("**/") { + let without_recursive_segment = format!( + "{}{}", + &pattern[..optional_recursive_start], + &pattern[optional_recursive_start + "**/".len()..] + ); + compile_cache_asset_glob_patterns(&without_recursive_segment, compiled)?; + } + + Ok(()) +} + +/// Built-in cache-rule presets that operators can enable explicitly. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub enum CacheAssetPreset { + /// Next.js build output under `/_next/static/`. + #[serde(rename = "nextjs-static")] + NextJsStatic, +} + +impl CacheAssetPreset { + fn matches_path(self, path: &str) -> bool { + match self { + Self::NextJsStatic => path.starts_with("/_next/static/"), + } + } +} + +/// Cache visibility parsed from operator configuration. +#[derive(Debug, Default, Clone, Copy, Deserialize, Serialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub enum CachePolicyVisibility { + /// Public browser/cache visibility. + #[default] + Public, + /// Private browser visibility. + Private, +} + +impl From for CacheVisibility { + fn from(value: CachePolicyVisibility) -> Self { + match value { + CachePolicyVisibility::Public => Self::Public, + CachePolicyVisibility::Private => Self::Private, + } + } +} + +fn path_extension(path: &str) -> Option { + let filename = path.rsplit('/').next()?; + let (_, extension) = filename.rsplit_once('.')?; + (!extension.is_empty()).then(|| extension.to_ascii_lowercase()) +} + +/// Operator-selected filename fingerprint convention for an immutable custom rule. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub enum CacheAssetFingerprintStyle { + /// A hexadecimal suffix, such as `app.0123abcd.js`. + Hex, + /// An eight-character uppercase Base32 suffix, such as `app-VRTVD5R5.js`. + EsbuildBase32, + /// An eight-character `Base64URL` suffix, such as `index-BsELY24f.js`. + ViteBase64Url, +} + +impl CacheAssetFingerprintStyle { + fn matches_candidate(self, candidate: &str) -> bool { + match self { + Self::Hex => { + candidate.len() >= 8 + && candidate.chars().all(|ch| ch.is_ascii_hexdigit()) + && candidate.chars().any(|ch| ch.is_ascii_alphabetic()) + } + Self::EsbuildBase32 => { + candidate.len() == 8 + && candidate + .chars() + .all(|ch| ch.is_ascii_uppercase() || matches!(ch, '2'..='7')) + && candidate.chars().any(|ch| ch.is_ascii_alphabetic()) + } + Self::ViteBase64Url => { + candidate.len() == 8 + && candidate + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_')) + && candidate.chars().any(|ch| ch.is_ascii_uppercase()) + && candidate.chars().any(|ch| { + ch.is_ascii_lowercase() || ch.is_ascii_digit() || matches!(ch, '-' | '_') + }) + } + } + } +} + +fn filename_contains_fingerprint(path: &str, style: CacheAssetFingerprintStyle) -> bool { + let filename = path.rsplit('/').next().unwrap_or(path); + let Some((stem, extension)) = filename.rsplit_once('.') else { + return false; + }; + if stem.is_empty() || extension.is_empty() { + return false; + } + + stem.char_indices() + .filter(|(_, ch)| matches!(ch, '.' | '-' | '_' | '~')) + .any(|(separator_index, separator)| { + let candidate_start = separator_index + separator.len_utf8(); + let prefix = &stem[..separator_index]; + let candidate = &stem[candidate_start..]; + !prefix.is_empty() && style.matches_candidate(candidate) + }) +} + /// Debug-only features. All flags default to `false` (off in production). #[derive(Debug, Default, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -1878,16 +2351,18 @@ pub struct DebugConfig { #[serde(default)] pub ja4_endpoint_enabled: bool, - /// Inject a `` HTML comment before `` dumping a - /// redacted per-provider auction result: pipeline stats (SSP count, mediator - /// status, winning bid count) plus every provider response — each bid's - /// creative previewed (not the full `adm` markup) and provider metadata - /// filtered to a fail-closed allowlist that drops identity-bearing keys. - /// Never enable in production — visible in page source and injects (bounded) - /// raw HTML from SSPs. + /// Inject a `` HTML comment before `` dumping + /// per-provider auction diagnostics. The default validates response-level + /// metadata, but bid fields and bounded creative previews remain visible; + /// this is not a fully anonymized dump. Never enable in production. #[serde(default)] pub auction_html_comment: bool, + /// Content and verbosity of the `auction_html_comment` dump. Ignored + /// when `auction_html_comment` is false. + #[serde(default)] + pub auction_html_comment_options: AuctionDebugCommentOptions, + /// Enable the testing-only direct GAM-replace path and the verbose per-bid /// `debug_bid` blob in `window.tsjs.bids`. /// @@ -1903,6 +2378,135 @@ pub struct DebugConfig { pub inject_adm_for_testing: bool, } +/// Metadata keys safe to surface in the `ts-debug` auction comment. +/// +/// Fail-closed superset: any key not listed here — notably `debug`, which +/// carries the resolved `OpenRTB` request (EC ID, `user.ext.eids`, the TC +/// consent string, `device.ip`, `device.geo`) plus per-bidder `httpcalls` — +/// is dropped in [`AuctionDebugCommentVerbosity::Redacted`] mode regardless +/// of what an operator lists in [`AuctionDebugCommentOptions::metadata_keys`]. +/// `metadata_keys` is a subset selector against this const, never a way to +/// add new keys. +pub(crate) const AUCTION_DEBUG_METADATA_ALLOWLIST: &[&str] = + &["error_type", "http_status", "message"]; + +/// Provider-controlled diagnostic keys exposed only by `Upstream` or `Full`. +/// +/// Values remain untyped upstream JSON and may contain request or identity +/// data. Keeping this list separate prevents [`AuctionDebugCommentOptions::metadata_keys`] +/// from widening the default response-metadata boundary. +pub(crate) const AUCTION_DEBUG_UPSTREAM_METADATA_KEYS: &[&str] = &[ + "errors", + "warnings", + "responsetimemillis", + "bidstatus", + "upstream_message", + "upstream_message_truncated", +]; + +fn default_true() -> bool { + true +} + +fn default_auction_debug_metadata_keys() -> Vec { + AUCTION_DEBUG_METADATA_ALLOWLIST + .iter() + .map(std::string::ToString::to_string) + .collect() +} + +/// Behavior of the `` auction dump. Only consulted when +/// [`DebugConfig::auction_html_comment`] is true. +/// +/// `deny_unknown_fields` matches the convention used by sibling config +/// structs in this file, including the `DebugConfig` this struct nests +/// under: an operator typo (e.g. `metadata_key` instead of `metadata_keys`) +/// must fail config load loudly, not be silently ignored. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AuctionDebugCommentOptions { + /// Include the `provider_responses` section at all. + #[serde(default = "default_true")] + pub include_provider_responses: bool, + + /// Include `mediator_response` when a mediator ran. + #[serde(default = "default_true")] + pub include_mediator_response: bool, + + /// Include each provider's `bids` array (vs. status/metadata only). + #[serde(default = "default_true")] + pub include_bids: bool, + + /// Subset of [`AUCTION_DEBUG_METADATA_ALLOWLIST`] to surface in + /// [`AuctionDebugCommentVerbosity::Redacted`] mode. Keys outside the + /// fixed allowlist are always dropped, config or not. This selector cannot + /// unlock provider diagnostics. Ignored when `verbosity` is `Full`. + #[serde(default = "default_auction_debug_metadata_keys")] + pub metadata_keys: Vec, + + /// `Redacted` (default): validated `metadata_keys` subset only, with + /// creative previews truncated to `MAX_BID_CREATIVE_DUMP_BYTES`. + /// `Upstream`: redacted fields plus six untyped provider diagnostics; + /// creative previews remain truncated. + /// `Full`: raw `response.metadata` verbatim, including the `debug` + /// subtree (httpcalls/resolvedrequest) when present, and no creative + /// truncation. The total dump byte cap and comment-terminator + /// neutralization still apply unconditionally. + /// + /// NEVER enable `Upstream` or `Full` in production — identity-bearing + /// request/response data may become visible via view-source. + #[serde(default)] + pub verbosity: AuctionDebugCommentVerbosity, + + /// JSON representation used for the outer auction dump. + #[serde(default)] + pub format: AuctionDebugCommentFormat, +} + +impl Default for AuctionDebugCommentOptions { + fn default() -> Self { + Self { + include_provider_responses: true, + include_mediator_response: true, + include_bids: true, + metadata_keys: default_auction_debug_metadata_keys(), + verbosity: AuctionDebugCommentVerbosity::Redacted, + format: AuctionDebugCommentFormat::Compact, + } + } +} + +impl AuctionDebugCommentOptions { + pub(crate) fn normalize(&mut self) { + self.metadata_keys = self + .metadata_keys + .drain(..) + .map(|key| key.trim().to_string()) + .filter(|key| !key.is_empty()) + .collect(); + } +} + +/// Verbosity of the `ts-debug` auction comment. See +/// [`AuctionDebugCommentOptions::verbosity`]. +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuctionDebugCommentVerbosity { + #[default] + Redacted, + Upstream, + Full, +} + +/// JSON representation used for the outer `ts-debug` auction dump. +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuctionDebugCommentFormat { + #[default] + Compact, + Pretty, +} + /// Tester-cookie endpoint configuration. #[derive(Debug, Default, Clone, Deserialize, Serialize)] pub struct TesterCookieConfig { @@ -1937,6 +2541,8 @@ pub struct Settings { #[serde(default)] pub consent: ConsentConfig, #[serde(default)] + pub cache: CacheSettings, + #[serde(default)] pub proxy: Proxy, #[serde(default)] pub creative_opportunities: Option, @@ -2019,8 +2625,10 @@ impl Settings { mut settings: Self, validation_label: &str, ) -> Result> { + settings.cache.normalize(); settings.proxy.normalize(); settings.image_optimizer.normalize(); + settings.debug.auction_html_comment_options.normalize(); settings.consent.validate(); settings.prepare_runtime()?; @@ -2052,6 +2660,7 @@ impl Settings { /// opportunity slot is invalid. pub fn prepare_runtime(&mut self) -> Result<(), Report> { self.image_optimizer.prepare_runtime()?; + self.cache.prepare_runtime()?; self.proxy.prepare_runtime()?; self.tinybird.prepare_runtime()?; self.validate_asset_image_optimizer_profile_sets()?; @@ -2095,13 +2704,14 @@ impl Settings { Ok(()) } - /// Returns compiled creative opportunity slots, or empty slice if feature is disabled. + /// Returns compiled creative opportunity slots when template delivery is enabled. #[must_use] pub fn creative_opportunity_slots( &self, ) -> &[crate::creative_opportunities::CreativeOpportunitySlot] { self.creative_opportunities .as_ref() + .filter(|co| co.enabled) .map(|co| co.slot.as_slice()) .unwrap_or(&[]) } @@ -2167,6 +2777,18 @@ impl Settings { Ok(()) } + /// Resolve the first matching configured asset cache policy for the request path. + /// + /// # Errors + /// + /// Returns a configuration error if matcher preparation unexpectedly fails. + pub fn asset_cache_policy_for_path( + &self, + path: &str, + ) -> Result, Report> { + self.cache.asset_policy_for_path(path) + } + /// Resolve the longest matching asset route for the request path. #[must_use] pub fn asset_route_for_path(&self, path: &str) -> Option<&ProxyAssetRoute> { @@ -2197,9 +2819,19 @@ impl Settings { /// where any of these paths lack a matching handler, ensuring admin /// endpoints are always protected by authentication. /// Update [`ADMIN_ENDPOINTS`](Self::ADMIN_ENDPOINTS) when adding new - /// admin routes to `crates/trusted-server-adapter-fastly/src/main.rs`. - pub(crate) const ADMIN_ENDPOINTS: &[&str] = - &["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate"]; + /// admin routes to `crates/trusted-server-adapter-fastly/src/app.rs`. + /// + /// The `/_ts/admin/ec/{id}` entry is the literal router pattern; handler + /// path regexes are matched against it verbatim, so prefix-style admin + /// regexes (e.g. `^/_ts/admin`) cover it while regexes too narrow to + /// cover the parameterized route are rejected fail-closed. + pub(crate) const ADMIN_ENDPOINTS: &[&str] = &[ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/_ts/admin/ec", + "/_ts/admin/ec/{id}", + "/_ts/admin/eids", + ]; /// Returns admin endpoint paths that no configured handler covers. /// @@ -2595,6 +3227,89 @@ mod tests { use crate::redacted::Redacted; use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; + #[test] + fn auction_debug_comment_options_default_matches_serde_defaults() { + let opts = AuctionDebugCommentOptions::default(); + assert!(opts.include_provider_responses, "should default to true"); + assert!(opts.include_mediator_response, "should default to true"); + assert!(opts.include_bids, "should default to true"); + assert_eq!( + opts.metadata_keys, + vec![ + "error_type".to_string(), + "http_status".to_string(), + "message".to_string(), + ], + "should default to only schema-validated response metadata" + ); + assert_eq!( + opts.verbosity, + AuctionDebugCommentVerbosity::Redacted, + "should default to Redacted" + ); + assert_eq!( + opts.format, + AuctionDebugCommentFormat::Compact, + "should default to compact output" + ); + } + + #[test] + fn auction_debug_comment_options_normalize_trims_and_drops_empty_keys() { + let mut opts = AuctionDebugCommentOptions { + metadata_keys: vec![ + " http_status ".to_string(), + "".to_string(), + "debug".to_string(), + ], + ..AuctionDebugCommentOptions::default() + }; + opts.normalize(); + assert_eq!( + opts.metadata_keys, + vec!["http_status".to_string(), "debug".to_string()] + ); + } + + #[test] + fn auction_debug_comment_options_deserializes_upstream_verbosity() { + let options: AuctionDebugCommentOptions = toml::from_str(r#"verbosity = "upstream""#) + .expect("should deserialize upstream verbosity"); + assert_eq!(options.verbosity, AuctionDebugCommentVerbosity::Upstream); + } + + #[test] + fn auction_debug_comment_options_deserializes_pretty_format() { + let options: AuctionDebugCommentOptions = + toml::from_str(r#"format = "pretty""#).expect("should deserialize pretty format"); + assert_eq!(options.format, AuctionDebugCommentFormat::Pretty); + } + + #[test] + fn auction_debug_comment_options_bad_format_fails_config_load() { + let result: Result = + toml::from_str(r#"format = "expanded""#); + assert!( + result.is_err(), + "unrecognized format must fail to deserialize, not silently fall back" + ); + } + + #[test] + fn bad_verbosity_string_fails_config_load() { + // Deserialize AuctionDebugCommentOptions directly, not a full Settings — + // Settings has required fields with no #[serde(default)] (e.g. + // `publisher`), so a full-Settings fixture missing them would fail with + // "missing field `publisher`" regardless of whether `verbosity` itself + // deserialized correctly, testing the wrong thing. + let result: Result = + toml::from_str(r#"verbosity = "everything""#); + assert!( + result.is_err(), + "unrecognized verbosity must fail to deserialize, not silently fall back" + ); + } + #[test] fn tinybird_defaults_to_disabled_placeholders() { let settings = Settings::from_toml(&crate_test_settings_str()) @@ -2729,6 +3444,418 @@ mod tests { ); } + #[test] + fn cache_asset_rule_nextjs_preset_is_operator_controlled() { + let toml_str = format!( + r#"{} + + [[cache.asset_rules]] + id = "nextjs-static" + enabled = true + preset = "nextjs-static" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse cache asset rule"); + + let policy = settings + .asset_cache_policy_for_path("/_next/static/chunks/app.js") + .expect("should evaluate cache rules") + .expect("should match enabled Next.js preset"); + assert_eq!( + policy, + CachePolicy::public_immutable(Duration::from_secs(31_536_000)), + "enabled preset should produce immutable static policy" + ); + + let disabled_toml = toml_str.replace("enabled = true", "enabled = false"); + let disabled_settings = + Settings::from_toml(&disabled_toml).expect("should parse disabled cache asset rule"); + assert!( + disabled_settings + .asset_cache_policy_for_path("/_next/static/chunks/app.js") + .expect("should evaluate disabled cache rules") + .is_none(), + "disabled preset must not mark framework paths immutable" + ); + } + + #[test] + fn cache_asset_rule_requires_selected_fingerprint_style() { + let expected_policy = CachePolicy::public_immutable(Duration::from_secs(31_536_000)); + for (style, matching_path, non_matching_path) in [ + ("hex", "/assets/app.0123abcd.js", "/assets/app-VRTVD5R5.js"), + ( + "esbuild-base32", + "/assets/app-VRTVD5R5.js", + "/assets/index-BsELY24f.js", + ), + ( + "vite-base64-url", + "/assets/index-BsELY24f.js", + "/assets/app.0123abcd.js", + ), + ] { + let toml_str = format!( + r#"{} + + [[cache.asset_rules]] + id = "publisher-assets" + enabled = true + path_globs = ["/assets/**/*.js"] + fingerprint_style = "{style}" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse cache asset rule"); + + assert_eq!( + settings + .asset_cache_policy_for_path(matching_path) + .expect("should evaluate cache rules"), + Some(expected_policy), + "{style} should match its configured fingerprint convention" + ); + assert!( + settings + .asset_cache_policy_for_path(non_matching_path) + .expect("should evaluate cache rules") + .is_none(), + "{style} should not fall through to another fingerprint convention" + ); + } + } + + #[test] + fn filename_fingerprint_gate_matches_only_the_selected_style() { + for (style, path, expected) in [ + ( + CacheAssetFingerprintStyle::Hex, + "/assets/app.0123abcd.js", + true, + ), + ( + CacheAssetFingerprintStyle::Hex, + "/assets/hero-Portrait.jpg", + false, + ), + ( + CacheAssetFingerprintStyle::EsbuildBase32, + "/assets/app-VRTVD5R5.js", + true, + ), + ( + CacheAssetFingerprintStyle::EsbuildBase32, + "/assets/hero-Portrait.jpg", + false, + ), + ( + CacheAssetFingerprintStyle::ViteBase64Url, + "/assets/index-BsELY24f.js", + true, + ), + ( + CacheAssetFingerprintStyle::ViteBase64Url, + "/assets/hero-Portrait.jpg", + true, + ), + ( + CacheAssetFingerprintStyle::ViteBase64Url, + "/assets/app.js", + false, + ), + ( + CacheAssetFingerprintStyle::ViteBase64Url, + "/assets/deadbeef.js", + false, + ), + ] { + assert_eq!( + filename_contains_fingerprint(path, style), + expected, + "{style:?} fingerprint result should match for {path}" + ); + } + } + + #[test] + fn cache_asset_rule_globs_respect_path_separators() { + let toml_str = format!( + r#"{} + + [[cache.asset_rules]] + id = "direct-assets" + enabled = true + path_glob = "/assets/*.js" + browser_ttl_seconds = 300 + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse cache asset rule"); + + assert!( + settings + .asset_cache_policy_for_path("/assets/app.js") + .expect("should evaluate direct asset rule") + .is_some(), + "single-star glob should match a direct child" + ); + for path in ["/assets/vendor/app.js", "/assets/app.JS"] { + assert!( + settings + .asset_cache_policy_for_path(path) + .expect("should evaluate direct asset rule") + .is_none(), + "single-star glob should not match {path}" + ); + } + + let recursive_toml = toml_str.replace("/assets/*.js", "/assets/**/*.js"); + let recursive_settings = + Settings::from_toml(&recursive_toml).expect("should parse recursive cache asset rule"); + for path in ["/assets/app.js", "/assets/vendor/app.js"] { + assert!( + recursive_settings + .asset_cache_policy_for_path(path) + .expect("should evaluate recursive asset rule") + .is_some(), + "double-star glob should match {path}" + ); + } + } + + #[test] + fn disabled_cache_asset_rules_defer_matcher_and_policy_validation() { + let toml_str = format!( + r#"{} + + [[cache.asset_rules]] + id = "disabled-invalid-regex" + enabled = false + path_regex = "[" + + [[cache.asset_rules]] + id = "disabled-placeholder" + enabled = false + + [[cache.asset_rules]] + id = "disabled-unsafe-immutable" + enabled = false + path_prefix = "/assets/" + immutable = true + "#, + crate_test_settings_str() + ); + + let settings = + Settings::from_toml(&toml_str).expect("should defer disabled rule validation"); + assert!( + settings + .asset_cache_policy_for_path("/assets/app-DA15JTLU.js") + .expect("should evaluate disabled cache rules") + .is_none(), + "disabled rules should never match" + ); + } + + #[test] + fn cache_asset_rule_policy_validation_rejects_unsafe_config() { + let missing_ttl = format!( + r#"{} + + [[cache.asset_rules]] + id = "missing-ttl" + enabled = true + path_prefix = "/assets/" + "#, + crate_test_settings_str() + ); + let missing_ttl_err = + Settings::from_toml(&missing_ttl).expect_err("should reject rule without a TTL"); + assert!( + format!("{missing_ttl_err:?}").contains("browser_ttl_seconds or edge_ttl_seconds"), + "should explain missing TTL: {missing_ttl_err:?}" + ); + + let immutable_without_fingerprint_style = format!( + r#"{} + + [[cache.asset_rules]] + id = "unsafe-immutable" + enabled = true + path_prefix = "/assets/" + browser_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + ); + let fingerprint_style_err = Settings::from_toml(&immutable_without_fingerprint_style) + .expect_err("should reject immutable rule without a fingerprint style"); + assert!( + format!("{fingerprint_style_err:?}").contains("fingerprint_style"), + "should explain immutable fingerprint-style requirement: {fingerprint_style_err:?}" + ); + + let immutable_without_browser_ttl = format!( + r#"{} + + [[cache.asset_rules]] + id = "immutable-without-browser-ttl" + enabled = true + path_prefix = "/assets/" + fingerprint_style = "hex" + browser_ttl_seconds = 0 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + ); + let browser_ttl_err = Settings::from_toml(&immutable_without_browser_ttl) + .expect_err("should reject immutable rule without positive browser TTL"); + assert!( + format!("{browser_ttl_err:?}").contains("positive browser_ttl_seconds"), + "should explain immutable browser TTL requirement: {browser_ttl_err:?}" + ); + + let private_edge_only = format!( + r#"{} + + [[cache.asset_rules]] + id = "private-edge-only" + enabled = true + path_prefix = "/assets/" + visibility = "private" + edge_ttl_seconds = 300 + "#, + crate_test_settings_str() + ); + let private_edge_only_err = Settings::from_toml(&private_edge_only) + .expect_err("should reject private rule with only an edge TTL"); + assert!( + format!("{private_edge_only_err:?}").contains("edge_ttl_seconds"), + "should explain that private rules cannot use an edge TTL: {private_edge_only_err:?}" + ); + + let private_dual_ttl = private_edge_only.replace( + "id = \"private-edge-only\"", + "id = \"private-dual-ttl\"\n browser_ttl_seconds = 300", + ); + let private_dual_ttl_err = Settings::from_toml(&private_dual_ttl) + .expect_err("should reject private rule with browser and edge TTLs"); + assert!( + format!("{private_dual_ttl_err:?}").contains("edge_ttl_seconds"), + "should reject edge TTL even when a private rule has a browser TTL: {private_dual_ttl_err:?}" + ); + + let private_browser_ttl = private_edge_only.replace( + "id = \"private-edge-only\"\n enabled = true\n path_prefix = \"/assets/\"\n visibility = \"private\"\n edge_ttl_seconds = 300", + "id = \"private-browser-ttl\"\n enabled = true\n path_prefix = \"/assets/\"\n visibility = \"private\"\n browser_ttl_seconds = 300", + ); + let private_settings = Settings::from_toml(&private_browser_ttl) + .expect("should accept a private rule with a browser TTL"); + let private_policy = private_settings + .asset_cache_policy_for_path("/assets/app.js") + .expect("should evaluate private cache rule") + .expect("should match private cache rule"); + assert_eq!( + private_policy + .cache_control_value(crate::cache_policy::EdgeCacheHeader::SurrogateControl), + "private, max-age=300", + "private rules should render their browser TTL" + ); + assert_eq!( + private_policy + .edge_header_value(crate::cache_policy::EdgeCacheHeader::SurrogateControl), + None, + "private rules should not render an edge cache TTL" + ); + } + + #[test] + fn cache_asset_rule_validation_rejects_invalid_config() { + let duplicate_ids = format!( + r#"{} + + [[cache.asset_rules]] + id = "duplicate" + enabled = true + path_prefix = "/assets/" + + [[cache.asset_rules]] + id = "duplicate" + enabled = true + path_prefix = "/static/" + "#, + crate_test_settings_str() + ); + let duplicate_err = + Settings::from_toml(&duplicate_ids).expect_err("should reject duplicate rule ids"); + assert!( + format!("{duplicate_err:?}").contains("duplicate id"), + "should explain duplicate rule id: {duplicate_err:?}" + ); + + let invalid_regex = format!( + r#"{} + + [[cache.asset_rules]] + id = "bad-regex" + enabled = true + path_regex = "[" + "#, + crate_test_settings_str() + ); + let regex_err = + Settings::from_toml(&invalid_regex).expect_err("should reject invalid regex"); + assert!( + format!("{regex_err:?}").contains("path_regex"), + "should explain invalid regex: {regex_err:?}" + ); + + let invalid_shape = format!( + r#"{} + + [[cache.asset_rules]] + id = "too-many-matchers" + enabled = true + path_prefix = "/assets/" + extensions = ["js"] + "#, + crate_test_settings_str() + ); + let shape_err = + Settings::from_toml(&invalid_shape).expect_err("should reject invalid matcher shape"); + assert!( + format!("{shape_err:?}").contains("exactly one matcher"), + "should explain invalid matcher shape: {shape_err:?}" + ); + + let missing_matcher = format!( + r#"{} + + [[cache.asset_rules]] + id = "missing-matcher" + enabled = true + browser_ttl_seconds = 60 + "#, + crate_test_settings_str() + ); + let missing_matcher_err = + Settings::from_toml(&missing_matcher).expect_err("should reject missing matcher"); + assert!( + format!("{missing_matcher_err:?}").contains("exactly one matcher"), + "should explain missing matcher: {missing_matcher_err:?}" + ); + } + #[test] fn validate_rejects_trailing_slash_in_origin_url() { let toml_str = crate_test_settings_str().replace( @@ -4874,7 +6001,13 @@ origin_host_header_overide = "www.example.com""#, .expect("should check admin coverage"); assert_eq!( uncovered, - vec!["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate"], + vec![ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/_ts/admin/ec", + "/_ts/admin/ec/{id}", + "/_ts/admin/eids", + ], "should report every admin endpoint as uncovered" ); } @@ -4908,7 +6041,12 @@ origin_host_header_overide = "www.example.com""#, .expect("should check admin coverage"); assert_eq!( uncovered, - vec!["/_ts/admin/keys/deactivate"], + vec![ + "/_ts/admin/keys/deactivate", + "/_ts/admin/ec", + "/_ts/admin/ec/{id}", + "/_ts/admin/eids", + ], "should detect the admin endpoints not covered by the narrow handler" ); } @@ -5014,6 +6152,10 @@ formats = [{ width = 300, height = 250 }] let co = settings .creative_opportunities .expect("should have creative_opportunities"); + assert!( + co.enabled, + "creative-opportunity templates should default to enabled" + ); assert_eq!(co.gam_network_id, "21765378893"); assert_eq!(co.auction_timeout_ms, Some(500)); assert_eq!( @@ -5023,6 +6165,45 @@ formats = [{ width = 300, height = 250 }] ); } + #[test] + fn settings_disables_creative_opportunity_slots_when_configured_off() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = false\ngam_network_id = \"21765378893\"\n\n[[creative_opportunities.slot]]\nid = \"atf\"\npage_patterns = [\"/\"]\nformats = [{{ width = 300, height = 250 }}]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("should parse disabled templates"); + assert!( + settings.creative_opportunity_slots().is_empty(), + "disabled template delivery should expose no runtime slots" + ); + } + + #[test] + fn settings_creative_opportunity_enabled_flag_supports_environment_override() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = true\ngam_network_id = \"21765378893\"\n", + crate_test_settings_str() + ); + let env_key = format!( + "{}{}CREATIVE_OPPORTUNITIES{}ENABLED", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + + temp_env::with_var(env_key, Some("false"), || { + let settings = Settings::from_toml_and_env(&toml) + .expect("should parse template enabled environment override"); + assert!( + !settings + .creative_opportunities + .expect("should have creative opportunities") + .enabled, + "environment override should disable template delivery" + ); + }); + } + #[test] fn settings_rejects_invalid_creative_opportunity_slot_id() { let toml = r#" diff --git a/crates/trusted-server-core/src/tsjs.rs b/crates/trusted-server-core/src/tsjs.rs index 133e6d011..c8aa71f03 100644 --- a/crates/trusted-server-core/src/tsjs.rs +++ b/crates/trusted-server-core/src/tsjs.rs @@ -1,4 +1,4 @@ -use trusted_server_js::{all_module_ids, concatenated_hash, single_module_hash}; +use trusted_server_js::{concatenated_hash, single_module_hash}; /// `/static` URL for the tsjs bundle with cache-busting hash based on /// the concatenated content of the given module set. @@ -11,31 +11,48 @@ pub fn tsjs_script_src(module_ids: &[&str]) -> String { /// `", - tsjs_script_src(module_ids) + "", + tsjs_script_src(module_ids), ) } -/// `/static` URL for the unified bundle with a conservative cache-busting hash. +/// `/static` URL for the unified bundle when exact module IDs are unavailable. +/// +/// This intentionally omits `?v=` because the serving path can only mark a URL +/// immutable when the hash matches the exact enabled module set. Use +/// [`tsjs_script_src`] with exact module IDs when [`IntegrationRegistry`] is +/// available. /// -/// Hashes all compiled module IDs so the cache invalidates whenever any module -/// changes. Over-invalidates slightly (includes deferred modules in the hash) -/// but never serves stale content. Use [`tsjs_script_src`] with exact module -/// IDs when `IntegrationRegistry` is available. +/// [`IntegrationRegistry`]: crate::integrations::IntegrationRegistry #[must_use] pub fn tsjs_unified_script_src() -> String { - let ids = all_module_ids(); - tsjs_script_src(&ids) + "/static/tsjs=tsjs-unified.min.js".to_string() } -/// `", + tsjs_unified_script_src() + ) } /// `/static` URL for one module with its own cache-busting hash. @@ -171,18 +188,21 @@ mod tests { } #[test] - fn tsjs_unified_helpers_use_all_module_ids() { - let ids = all_module_ids(); + fn publisher_tsjs_script_tag_renders_static_attributes() { + let module_ids = ["gpt"]; + let src = tsjs_script_src(&module_ids); assert_eq!( - tsjs_unified_script_src(), - tsjs_script_src(&ids), - "should hash all module IDs for the unified script source" + tsjs_script_tag_with_attributes(&module_ids, &[("data-ts-gam-attribution", "true")]), + format!( + "" + ), + "should render trusted static attributes on the publisher bundle tag" ); assert_eq!( - tsjs_unified_script_tag(), - tsjs_script_tag(&ids), - "should wrap the all-module unified script source" + tsjs_script_tag(&module_ids), + format!(""), + "should keep the generic tag byte-for-byte unmarked" ); } @@ -246,14 +266,13 @@ mod tests { } #[test] - fn tsjs_unified_script_src_and_tag_include_cache_busting_hash() { + fn tsjs_unified_script_src_and_tag_omit_unverifiable_cache_busting_hash() { let src = tsjs_unified_script_src(); - assert!( - src.starts_with("/static/tsjs=tsjs-unified.min.js?v="), - "should include unified script URL prefix" + assert_eq!( + src, "/static/tsjs=tsjs-unified.min.js", + "should use the unified script URL without an unverifiable hash" ); - assert_sha256_hex_hash(hash_query_value(&src)); assert_eq!( tsjs_unified_script_tag(), format!(r#""#), diff --git a/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts b/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts index 3a38aa746..78059596f 100644 --- a/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts +++ b/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts @@ -9,6 +9,7 @@ test.describe("Script injection", () => { const src = await scriptTag.getAttribute("src"); expect(src).toContain("/static/tsjs="); + await expect(scriptTag).not.toHaveAttribute("data-ts-gam-attribution"); }); test("no unexpected console errors on page load", async ({ page }) => { diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml index 17d7c2713..d8e35d179 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml @@ -86,6 +86,7 @@ rewrite_sdk = true [integrations.gpt] enabled = false +gam_attribution_enabled = false script_url = "https://ads.example.com/gpt.js" cache_ttl_seconds = 3600 rewrite_script = true diff --git a/crates/trusted-server-integration-tests/tests/common/ec.rs b/crates/trusted-server-integration-tests/tests/common/ec.rs index cde6ad1c4..0a1f149c3 100644 --- a/crates/trusted-server-integration-tests/tests/common/ec.rs +++ b/crates/trusted-server-integration-tests/tests/common/ec.rs @@ -403,3 +403,79 @@ impl Drop for MinimalOrigin { } } } + +/// A minimal HTTP origin that reflects the request's Cookie header in a +/// cacheable HTML response. +/// +/// This makes it possible to assert that an edge runtime does not reuse a +/// cookie-influenced publisher response for another visitor. +pub struct CookieVaryingOrigin { + shutdown_tx: mpsc::Sender<()>, + handle: Option>, +} + +impl CookieVaryingOrigin { + /// Starts the cookie-varying origin on `127.0.0.1:{port}`. + /// + /// # Panics + /// + /// Panics if the port is already in use. + pub fn start(port: u16) -> Self { + let listener = + TcpListener::bind(format!("127.0.0.1:{port}")).expect("should bind origin port"); + listener + .set_nonblocking(true) + .expect("should set listener nonblocking"); + let (shutdown_tx, shutdown_rx) = mpsc::channel::<()>(); + + let handle = thread::spawn(move || { + loop { + if shutdown_rx.try_recv().is_ok() { + break; + } + + match listener.accept() { + Ok((mut stream, _addr)) => { + let mut buf = [0u8; 4096]; + let Ok(bytes_read) = stream.read(&mut buf) else { + continue; + }; + let request = String::from_utf8_lossy(&buf[..bytes_read]); + let cookie = request + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("cookie").then(|| value.trim()) + }) + .unwrap_or("viewer=missing"); + let body = format!("{cookie}"); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); + } + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(_) => break, + } + } + }); + + Self { + shutdown_tx, + handle: Some(handle), + } + } +} + +impl Drop for CookieVaryingOrigin { + fn drop(&mut self) { + let _ = self.shutdown_tx.send(()); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} diff --git a/crates/trusted-server-integration-tests/tests/integration.rs b/crates/trusted-server-integration-tests/tests/integration.rs index 76a267d1f..ebd026410 100644 --- a/crates/trusted-server-integration-tests/tests/integration.rs +++ b/crates/trusted-server-integration-tests/tests/integration.rs @@ -2,6 +2,7 @@ mod common; mod environments; mod frameworks; +use common::ec::CookieVaryingOrigin; use common::runtime::{RuntimeEnvironment, TestError, origin_port, wasm_binary_path}; use environments::{RUNTIME_ENVIRONMENTS, ReadyCheckOptions, wait_for_http_ready}; use error_stack::ResultExt as _; @@ -164,6 +165,55 @@ fn test_nextjs_cloudflare() { test_combination(&runtime, &framework).expect("should pass Next.js on Cloudflare Workers"); } +#[test] +#[ignore = "requires the `wrangler` CLI in $PATH and a prebuilt Cloudflare Workers bundle (run build.sh first); the test starts wrangler dev automatically"] +fn test_cloudflare_dynamic_publisher_response_does_not_cross_cookie_boundaries() { + init_logger(); + let _origin = CookieVaryingOrigin::start(origin_port()); + let runtime = environments::cloudflare::CloudflareWorkers; + let process = runtime + .spawn(&wasm_binary_path()) + .expect("should start Cloudflare Worker"); + let client = reqwest::blocking::Client::new(); + + let first_response = client + .get(format!("{}/cache-regression", process.base_url)) + .header("cookie", "viewer=first") + .send() + .expect("should request first dynamic publisher response"); + assert_eq!( + first_response.status().as_u16(), + 200, + "first dynamic publisher response should succeed" + ); + let first_body = first_response + .text() + .expect("should read first dynamic publisher response"); + + let second_response = client + .get(format!("{}/cache-regression", process.base_url)) + .header("cookie", "viewer=second") + .send() + .expect("should request second dynamic publisher response"); + assert_eq!( + second_response.status().as_u16(), + 200, + "second dynamic publisher response should succeed" + ); + let second_body = second_response + .text() + .expect("should read second dynamic publisher response"); + + assert!( + first_body.contains("viewer=first"), + "first response must preserve its cookie-specific origin body: {first_body}" + ); + assert!( + second_body.contains("viewer=second"), + "second response must not reuse the first visitor's body: {second_body}" + ); +} + #[test] #[ignore = "requires Docker and pre-built trusted-server-axum binary"] fn test_wordpress_axum() { diff --git a/crates/trusted-server-js/Cargo.toml b/crates/trusted-server-js/Cargo.toml index f3af9bfcf..67a4ac698 100644 --- a/crates/trusted-server-js/Cargo.toml +++ b/crates/trusted-server-js/Cargo.toml @@ -15,10 +15,11 @@ workspace = true doctest = false name = "trusted_server_js" path = "src/lib.rs" -test = false [build-dependencies] build-print = { workspace = true } +hex = { workspace = true } +sha2 = { workspace = true } which = { workspace = true } [dependencies] diff --git a/crates/trusted-server-js/build.rs b/crates/trusted-server-js/build.rs index 6d6bdde9f..ba6cd88f2 100644 --- a/crates/trusted-server-js/build.rs +++ b/crates/trusted-server-js/build.rs @@ -12,6 +12,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, ExitStatus}; use build_print::{info, warn}; +use sha2::{Digest as _, Sha256}; fn main() { // Rebuild if TS sources change (belt-and-suspenders): enumerate every file under lib/ @@ -125,7 +126,7 @@ fn main() { // Copy each module file to OUT_DIR for (_, filename) in &modules { - copy_bundle(filename, true, &crate_dir, &dist_dir, &out_dir); + copy_bundle(filename, true, &dist_dir, &out_dir); } // Generate tsjs_modules.rs with include_str!() for each module @@ -139,9 +140,10 @@ fn main() { ) .expect("should write generated module header"); for (id, filename) in &modules { + let sha256 = bundle_sha256(&out_dir.join(filename)); writeln!( codegen, - " TsjsModuleMeta {{\n bundle: include_str!(concat!(env!(\"OUT_DIR\"), \"/{filename}\")),\n id: \"{id}\",\n }},\n" + " TsjsModuleMeta {{\n bundle: include_str!(concat!(env!(\"OUT_DIR\"), \"/{filename}\")),\n id: \"{id}\",\n sha256: \"{sha256}\",\n }},\n" ) .expect("should write generated module entry"); } @@ -149,6 +151,7 @@ fn main() { codegen.push_str("\npub(crate) struct TsjsModuleMeta {\n"); codegen.push_str(" pub bundle: &'static str,\n"); codegen.push_str(" pub id: &'static str,\n"); + codegen.push_str(" pub sha256: &'static str,\n"); codegen.push_str("}\n"); let generated_path = out_dir.join("tsjs_modules.rs"); @@ -160,30 +163,36 @@ fn main() { }); } -fn copy_bundle(filename: &str, required: bool, crate_dir: &Path, dist_dir: &Path, out_dir: &Path) { - let primary = dist_dir.join(filename); - let fallback = crate_dir.join("dist").join(filename); +fn bundle_sha256(path: &Path) -> String { + let content = fs::read(path).unwrap_or_else(|err| { + panic!( + "tsjs: failed to read copied bundle {} for hashing: {err}", + path.display() + ); + }); + hex::encode(Sha256::digest(&content)) +} + +fn copy_bundle(filename: &str, required: bool, dist_dir: &Path, out_dir: &Path) { + let source = dist_dir.join(filename); let target = out_dir.join(filename); - for source in [&primary, &fallback] { - if source.exists() { - if let Err(err) = fs::copy(source, &target) { - assert!( - !required, - "tsjs: failed to copy {} to {}: {err}", - source.display(), - target.display() - ); - } - return; + if source.exists() { + if let Err(err) = fs::copy(&source, &target) { + assert!( + !required, + "tsjs: failed to copy {} to {}: {err}", + source.display(), + target.display() + ); } + return; } assert!( !required, - "tsjs: bundle {filename} not found: {} (and fallback {}). Ensure Node is installed and `npm run build` succeeds, or commit dist/{filename}.", - primary.display(), - fallback.display() + "tsjs: bundle {filename} not found: {}. Ensure Node is installed and `npm run build` succeeds, or commit dist/{filename}.", + source.display() ); fs::write(&target, "").expect("should write optional empty bundle placeholder"); diff --git a/crates/trusted-server-js/lib/src/core/index.ts b/crates/trusted-server-js/lib/src/core/index.ts index 5d8e41971..b2c4e41e1 100644 --- a/crates/trusted-server-js/lib/src/core/index.ts +++ b/crates/trusted-server-js/lib/src/core/index.ts @@ -36,10 +36,10 @@ api.getConfig = getConfig; // Provide core requestAds API api.requestAds = requestAds; // Defensive defaults: the edge injects adSlots (head-open) and bids (before -// ) only when the server-side ad stack runs for the request. When it -// is gated off (kill switch, consent fail-closed, bots, prefetch), page code -// reading window.tsjs.bids / window.tsjs.adSlots must still see defined -// values instead of throwing. Injected scripts overwrite these wholesale. +// ) only when server-side ad templates run for the request. When template +// delivery is disabled or gated off (auction/consent, bots, prefetch), page code +// reading window.tsjs.bids / window.tsjs.adSlots must still see defined values +// instead of throwing. Injected scripts overwrite these wholesale. api.adSlots ??= []; api.bids ??= {}; // Point global tsjs diff --git a/crates/trusted-server-js/lib/src/core/request.ts b/crates/trusted-server-js/lib/src/core/request.ts index df3fee6a6..c05070a9d 100644 --- a/crates/trusted-server-js/lib/src/core/request.ts +++ b/crates/trusted-server-js/lib/src/core/request.ts @@ -1,5 +1,5 @@ // Request orchestration for tsjs: unified auction endpoint with iframe-based creative rendering. -import { renderApsCreative } from '../integrations/aps/render'; +import { dispatchApsRendering, renderApsCreative } from '../integrations/aps/render'; import { buildAdRequest, sendAuction } from './auction'; import { collectContext } from './context'; @@ -52,7 +52,13 @@ export function requestAds( for (const bid of bids) { if (!bid.impid) continue; if (bid.renderer) { - renderApsCreative({ slotId: bid.impid, renderer: bid.renderer }); + void Promise.resolve( + dispatchApsRendering({ + slotId: bid.impid, + renderer: bid.renderer, + trustedServer: (renderer) => renderApsCreative({ slotId: bid.impid, renderer }), + }) + ); continue; } if (!bid.adm) { diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 0c68d43fe..43b3a1cf3 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -68,6 +68,21 @@ export interface ApsRendererV1 { export type AuctionBidRenderer = ApsRendererV1; +/** Explicit acknowledgement returned by the opt-in publisher-native APS hook. */ +export interface ApsNativeRendererResult { + accepted: boolean; + reason?: string; +} + +/** Publisher-owned rendering seam for a fully validated APS descriptor. */ +export interface ApsNativeRendererHook { + render(input: { + version: 1; + slotId: string; + renderer: ApsRendererV1; + }): ApsNativeRendererResult | Promise; +} + /** A client-side Prebid bid's generated ad ID bound to its APS render capability. */ export interface ApsPrebidRendererEntry { adUnitCode: string; @@ -207,7 +222,15 @@ export interface GptDiagnosticsRequestCycle { viewableAtMs?: number; durations: GptDiagnosticsDurations; isEmpty?: boolean; + /** Configured sizes Trusted Server supplied to GPT for this request. */ + requestedSlotSizes?: ReadonlyArray; + /** Exact fill size fact GPT reported in its `slotRenderEnded` callback. */ size?: Size; + /** + * Outer CSS box observed on the uniquely bound, connected slot element after + * a filled GPT render. This is not an assertion about internal creative pixels. + */ + observedSlotSize?: Size; isBackfill?: boolean; slotContentChanged?: boolean; incompleteSequence: boolean; @@ -318,12 +341,13 @@ export interface GptDiagnosticsApi { * and stops the writers from becoming part of the public contract. */ export interface GptDiagnosticsRecorder { - /** Record Trusted Server's creative opportunity for an associated GPT slot. */ + /** Record Trusted Server's creative opportunity and configured sizes for an associated GPT slot. */ recordTrustedServerOpportunity( slot: GptDiagnosticsSlotHandle, auctionSlotId: string, opportunity: GptDiagnosticsTrustedServerOpportunity, - trustedServerAuctionId?: string + trustedServerAuctionId?: string, + requestedSlotSizes?: ReadonlyArray ): void; /** Mark slots whose next observed GPT request follows the Prebid refresh path. */ recordPrebidRefresh(slots: GptDiagnosticsSlotHandle[]): void; @@ -388,6 +412,8 @@ export interface TsjsApi { * `hb_adid`. The Universal Creative bridge consumes each entry at most once. */ apsPrebidRenderers?: Record; + /** Opt-in publisher-owned renderer for exact, validated APS descriptors. */ + apsNativeRenderer?: ApsNativeRendererHook; /** Initialises GPT slots with server-side bid targeting and calls refresh(). */ adInit?: () => void; /** GPT slot objects TS defined — used to destroy stale slots on SPA navigation. */ @@ -452,8 +478,18 @@ export interface TsjsApi { * Lives in the bundle so the lifecycle is executable under test and shares * [`navGeneration`] with the SPA auction hook; `gpt_bootstrap.js` installs * a minimal fallback for pages where the bundle fails to load. + * + * `initialSlots` exists for the shared-template `` seam, which is the + * only place slot definitions arrive with the bids rather than from the head + * script. Passing them here rather than assigning `tsjs.adSlots` before the + * call puts them behind the same generation guard: an assignment made ahead + * of the guard would clobber a committed SPA navigation's slots with the SSR + * document's, and then be read by that route's `adInit()`. */ - scheduleInitialAdInit?: (initialBids?: Record) => void; + scheduleInitialAdInit?: ( + initialBids?: Record, + initialSlots?: AuctionSlot[] + ) => void; /** Read-only GPT lifecycle diagnostics API, present only in an activated tab. */ gptDiagnostics?: GptDiagnosticsApi; /** diff --git a/crates/trusted-server-js/lib/src/integrations/aps/render.ts b/crates/trusted-server-js/lib/src/integrations/aps/render.ts index 85f17adf9..f5faa60a1 100644 --- a/crates/trusted-server-js/lib/src/integrations/aps/render.ts +++ b/crates/trusted-server-js/lib/src/integrations/aps/render.ts @@ -1,7 +1,14 @@ import { log } from '../../core/log'; -import type { ApsPrebidRendererEntry, ApsRendererV1, TsjsApi } from '../../core/types'; +import type { + ApsNativeRendererHook, + ApsPrebidRendererEntry, + ApsRendererV1, + TsjsApi, +} from '../../core/types'; export const APS_RENDERER_PATH = '/integrations/aps/renderer'; +export const APS_RENDERING_MODE_META_NAME = 'trusted-server-aps-rendering-mode'; +export const APS_NATIVE_RENDERER_ACK_TIMEOUT_MS = 10_000; export const APS_RENDERER_SANDBOX = 'allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation'; export const APS_UNIVERSAL_CREATIVE_RENDERER_VERSION = 4; @@ -38,6 +45,13 @@ type ValidatedRendererCacheEntry = { renderer: ApsRendererV1; }; const validatedRendererCache = new WeakMap(); +const nativeDispatches = new Map(); + +function releaseNativeDispatch(slotId: string, dispatch: symbol): boolean { + if (nativeDispatches.get(slotId) !== dispatch) return false; + nativeDispatches.delete(slotId); + return true; +} function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); @@ -277,6 +291,133 @@ export function consumeApsPrebidRenderer(adId: string, expected: ApsPrebidRender return true; } +/** Whether the server explicitly selected the opt-in publisher-native hook mode. */ +export function isPublisherNativeApsRendering(): boolean { + return ( + document.head.querySelector( + `meta[name="${APS_RENDERING_MODE_META_NAME}"][content="publisher_native"]` + ) !== null + ); +} + +export interface DispatchApsRenderingOptions { + slotId: string; + renderer: unknown; + /** Existing Trusted Server owner, invoked only in the default mode. */ + trustedServer: (renderer: ApsRendererV1) => boolean; +} + +/** + * Dispatch a validated APS descriptor to exactly one configured rendering owner. + * + * Publisher hooks own side-effect cancellation and render completion. Trusted Server + * ignores superseded hook acknowledgements and never falls back to its iframe. + */ +export function dispatchApsRendering({ + slotId, + renderer: input, + trustedServer, +}: DispatchApsRenderingOptions): boolean | Promise { + // Record every attempt before any early return so it supersedes an older + // pending native acknowledgement for the same slot. + const dispatch = Symbol(slotId); + nativeDispatches.set(slotId, dispatch); + + const renderer = validateApsRenderer(input); + if (!renderer) { + releaseNativeDispatch(slotId, dispatch); + log.warn('APS renderer: rejected descriptor'); + return false; + } + if (!isPublisherNativeApsRendering()) { + try { + return trustedServer(renderer); + } finally { + releaseNativeDispatch(slotId, dispatch); + } + } + + let hook: ApsNativeRendererHook | undefined; + let render: ApsNativeRendererHook['render'] | undefined; + try { + hook = window.tsjs?.apsNativeRenderer; + render = hook?.render; + } catch { + releaseNativeDispatch(slotId, dispatch); + log.warn('APS native renderer: publisher hook lookup threw'); + return Promise.resolve(false); + } + if (!hook || typeof render !== 'function') { + releaseNativeDispatch(slotId, dispatch); + log.warn('APS native renderer: publisher hook is unavailable'); + return Promise.resolve(false); + } + + let response: unknown; + try { + response = Reflect.apply(render, hook, [{ version: 1, slotId, renderer }]); + } catch { + releaseNativeDispatch(slotId, dispatch); + log.warn('APS native renderer: publisher hook threw'); + return Promise.resolve(false); + } + + if (nativeDispatches.get(slotId) !== dispatch) { + log.warn('APS native renderer: ignored stale acknowledgement'); + return Promise.resolve(false); + } + + return new Promise((resolve) => { + let settled = false; + const settle = (accepted: boolean, warning?: string): void => { + if (settled) return; + settled = true; + clearTimeout(timeout); + if (!releaseNativeDispatch(slotId, dispatch)) { + log.warn('APS native renderer: ignored stale acknowledgement'); + resolve(false); + return; + } + if (warning) log.warn(warning); + resolve(accepted); + }; + const timeout = setTimeout(() => { + settle(false, 'APS native renderer: publisher hook acknowledgement timed out'); + }, APS_NATIVE_RENDERER_ACK_TIMEOUT_MS); + + Promise.resolve(response).then( + (value) => { + let accepted: boolean; + try { + if ( + !isRecord(value) || + (!hasExactKeys(value, ['accepted']) && !hasExactKeys(value, ['accepted', 'reason'])) || + typeof value.accepted !== 'boolean' || + (Object.prototype.hasOwnProperty.call(value, 'reason') && + typeof value.reason !== 'string') + ) { + settle(false, 'APS native renderer: publisher hook returned malformed acknowledgement'); + return; + } + accepted = value.accepted; + } catch { + settle(false, 'APS native renderer: publisher hook returned malformed acknowledgement'); + return; + } + + if (!accepted) { + settle(false, 'APS native renderer: publisher hook declined descriptor'); + return; + } + settle(true); + }, + () => { + settle(false, 'APS native renderer: publisher hook rejected'); + } + ); + }); +} + function createNonce(): string | undefined { if (typeof crypto === 'undefined' || typeof crypto.getRandomValues !== 'function') return undefined; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index f0df35974..10e62268e 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -11,6 +11,7 @@ import { APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, apsRendererUrl, + dispatchApsRendering, consumeApsPrebidRenderer, getApsPrebidRenderer, validateApsRenderer, @@ -287,6 +288,8 @@ type GptWindow = Window & { __tsjs_slim_prebid_url?: string; }; +const executingPublisherScript = typeof document === 'undefined' ? null : document.currentScript; + // ------------------------------------------------------------------ // Shim implementation // ------------------------------------------------------------------ @@ -307,6 +310,25 @@ function ensureGoogleTagStub(win: GptWindow): Partial { return tag; } +function installTrustedServerPageTargeting(): void { + if (executingPublisherScript?.getAttribute('data-ts-gam-attribution') !== 'true') { + return; + } + + const win = window as GptWindow; + const tag = ensureGoogleTagStub(win); + tag.cmd!.push(() => { + try { + const gpt = win.googletag; + if (typeof gpt?.setConfig === 'function') { + gpt.setConfig({ targeting: { ts: 'true' } }); + } + } catch (error) { + log.warn('[tsjs-gpt] GAM attribution targeting failed', error); + } + }); +} + /** * Wrap a queued GPT callback to add instrumentation and future hook points. * @@ -665,7 +687,15 @@ function installInitialLoadDetector(ts: TsjsApi): void { * SSR bootstrap as current. For the same reason the initial bids payload is * passed in and applied here, generation-guarded — assigning it * unconditionally at body end would clobber the live bids a faster SPA - * navigation already applied. When a navigation has committed since — or + * navigation already applied. + * + * `initialSlots` is passed in for exactly the same reason and was missing it. + * Only the shared-template `` seam sends slots — under `inline` they + * come from the head script, which runs before any navigation can commit — and + * that seam assigned `tsjs.adSlots` on the line *before* calling this. The + * guard protected the bids and `adInit()` while the assignment it was meant to + * protect had already happened, so a committed SPA navigation kept its bids and + * lost its slots. When a navigation has committed since — or * commits while the deferred callback is pending — the SSR payload is * dropped and `adInit()` is not run: running anyway would re-run the newer * route's live slots/bids, destroying and redefining that route's TS slots @@ -685,8 +715,12 @@ function installInitialLoadDetector(ts: TsjsApi): void { * holds whenever the request is actually issued. */ function installScheduleInitialAdInit(ts: TsjsApi): void { - ts.scheduleInitialAdInit = function (initialBids?: Record) { + ts.scheduleInitialAdInit = function ( + initialBids?: Record, + initialSlots?: AuctionSlot[] + ) { if ((ts.navGeneration ?? 0) !== 0) return; + if (initialSlots) ts.adSlots = initialSlots; if (initialBids) ts.bids = initialBids; const runUnlessNavigated = (): void => { if ((ts.navGeneration ?? 0) !== 0) return; @@ -1053,20 +1087,13 @@ export function installTsAdInit(): void { // implementation must never interrupt slot mapping or delivery. try { const opportunity = trustedServerOpportunity(bid); - if (bid.hb_auction_id !== undefined) { - ts.gptDiagnosticsRecorder?.recordTrustedServerOpportunity( - gptSlot, - slot.id, - opportunity, - bid.hb_auction_id - ); - } else { - ts.gptDiagnosticsRecorder?.recordTrustedServerOpportunity( - gptSlot, - slot.id, - opportunity - ); - } + ts.gptDiagnosticsRecorder?.recordTrustedServerOpportunity( + gptSlot, + slot.id, + opportunity, + bid.hb_auction_id, + slot.formats + ); } catch { // Diagnostics must not alter ad delivery. } @@ -1094,8 +1121,8 @@ export function installTsAdInit(): void { ts.prevSlotTargetingKeys = nextSlotTargetingKeys; // Whether this call produced any TS slot to render. A gated page-bids - // response (auction kill switch or consent denial) returns no slots, so - // the loops above leave these empty. + // response (template switch, auction gate, or consent denial) returns no + // slots, so the loops above leave these empty. const hasRenderableWork = slotsToDisplay.length > 0 || slotsToRefresh.length > 0; // enableSingleRequest and enableServices must only be called once per page @@ -1403,10 +1430,10 @@ export function installSpaAuctionHook(): void { // This route is now the committed, loaded state — a later failed // navigation rolls back here, and a return trip no-ops correctly. lastAppliedPath = path; - // An empty page-bids response (auction kill switch or consent gate) carries - // no TS slots. Only run adInit() when there are slots to apply or prior TS - // state to sweep — otherwise a consent-denied or kill-switched navigation - // must not enter the GPT command queue and risk activating services. + // An empty page-bids response (template switch, auction, or consent gate) + // carries no TS slots. Only run adInit() when there are slots to apply or + // prior TS state to sweep — otherwise a gated navigation must not enter + // the GPT command queue and risk activating services. const hasPriorTsState = (ts.prevGptSlots?.length ?? 0) > 0 || Object.keys(ts.prevSlotTargetingKeys ?? {}).length > 0 || @@ -1676,29 +1703,49 @@ export function installTsRenderBridge(): void { e.stopImmediatePropagation(); if (!messageSourceBelongsToAdUnit(e.source, prebidRendererEntry.adUnitCode)) return; const renderer = validateApsRenderer(prebidRendererEntry.renderer); - const rendererUrl = apsRendererUrl(); - if (!renderer || !rendererUrl) return; - if (!hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; + if (!renderer || !hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; if (!consumeApsPrebidRenderer(adId, prebidRendererEntry)) return; recordConsumedPrebidApsId(consumedPrebidApsIds, adId, prebidRendererEntry.expiresAt); - port.postMessage( - JSON.stringify({ - message: 'Prebid Response', - adId, - renderer: APS_UNIVERSAL_CREATIVE_RENDERER, - rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - rendererUrl, - apsRenderer: renderer, - width: renderer.width, - height: renderer.height, - }) - ); - - try { - prebidRendererEntry.markUsed(); - } catch (err) { - log.warn(`[tsjs-gpt] APS Prebid markUsed callback threw for '${adId}'`, err); + const markUsed = (): void => { + try { + prebidRendererEntry.markUsed(); + } catch (err) { + log.warn(`[tsjs-gpt] APS Prebid markUsed callback threw for '${adId}'`, err); + } + }; + const dispatched = dispatchApsRendering({ + slotId: prebidRendererEntry.adUnitCode, + renderer, + trustedServer: (validatedRenderer) => { + const rendererUrl = apsRendererUrl(); + if (!rendererUrl) return false; + try { + port.postMessage( + JSON.stringify({ + message: 'Prebid Response', + adId, + renderer: APS_UNIVERSAL_CREATIVE_RENDERER, + rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, + rendererUrl, + apsRenderer: validatedRenderer, + width: validatedRenderer.width, + height: validatedRenderer.height, + }) + ); + return true; + } catch (err) { + log.warn(`[tsjs-gpt] APS Prebid response post failed for '${adId}'`, err); + return false; + } + }, + }); + if (typeof dispatched === 'boolean') { + if (dispatched) markUsed(); + } else { + void dispatched.then((accepted) => { + if (accepted) markUsed(); + }); } return; } @@ -1727,19 +1774,34 @@ export function installTsRenderBridge(): void { e.stopImmediatePropagation(); if (consumedServerApsBySlot.get(slotId) === adId) return; const renderer = validateApsRenderer(matchedBid.renderer); - const rendererUrl = apsRendererUrl(); - if (!renderer || !rendererUrl) return; + if (!renderer) return; consumedServerApsBySlot.set(slotId, adId); - port.postMessage( - JSON.stringify({ - message: 'Prebid Response', - adId, - renderer: APS_UNIVERSAL_CREATIVE_RENDERER, - rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - rendererUrl, - apsRenderer: renderer, - width: renderer.width, - height: renderer.height, + void Promise.resolve( + dispatchApsRendering({ + slotId, + renderer, + trustedServer: (validatedRenderer) => { + const rendererUrl = apsRendererUrl(); + if (!rendererUrl) return false; + try { + port.postMessage( + JSON.stringify({ + message: 'Prebid Response', + adId, + renderer: APS_UNIVERSAL_CREATIVE_RENDERER, + rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, + rendererUrl, + apsRenderer: validatedRenderer, + width: validatedRenderer.width, + height: validatedRenderer.height, + }) + ); + return true; + } catch (err) { + log.warn(`[tsjs-gpt] APS server response post failed for '${slotId}'`, err); + return false; + } + }, }) ); return; @@ -1892,6 +1954,7 @@ if (typeof window !== 'undefined') { installGptShim(); } + installTrustedServerPageTargeting(); installTsAdInit(); installSpaAuctionHook(); installSlimPrebidLoader(); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts index 99f876b3f..475bc7f93 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts @@ -17,7 +17,8 @@ interface ApiStore { slot: GptDiagnosticsSlotHandle, auctionSlotId: string, opportunity: GptDiagnosticsTrustedServerOpportunity, - trustedServerAuctionId?: string + trustedServerAuctionId?: string, + requestedSlotSizes?: ReadonlyArray ): void; recordPrebidRefresh(slots: GptDiagnosticsSlotHandle[]): void; recordTrustedServerCreativeRequest(auctionSlotId: string): number | undefined; @@ -63,7 +64,9 @@ function cloneExportSnapshot(snapshot: GptDiagnosticsExportV1): GptDiagnosticsEx requests: slot.requests.map((cycle) => ({ ...cycle, durations: { ...cycle.durations }, + requestedSlotSizes: cycle.requestedSlotSizes?.map((size) => [...size]), size: cycle.size ? [...cycle.size] : undefined, + observedSlotSize: cycle.observedSlotSize ? [...cycle.observedSlotSize] : undefined, adManager: cycle.adManager ? { ...cycle.adManager, @@ -149,18 +152,21 @@ export class GptDiagnosticsApiController { }; this.recorder = { - recordTrustedServerOpportunity: (slot, auctionSlotId, opportunity, trustedServerAuctionId) => + recordTrustedServerOpportunity: ( + slot, + auctionSlotId, + opportunity, + trustedServerAuctionId, + requestedSlotSizes + ) => safelyRecord(() => { - if (trustedServerAuctionId === undefined) { - this.store.recordTrustedServerOpportunity(slot, auctionSlotId, opportunity); - } else { - this.store.recordTrustedServerOpportunity( - slot, - auctionSlotId, - opportunity, - trustedServerAuctionId - ); - } + this.store.recordTrustedServerOpportunity( + slot, + auctionSlotId, + opportunity, + trustedServerAuctionId, + requestedSlotSizes + ); }), recordPrebidRefresh: (slots) => safelyRecord(() => this.store.recordPrebidRefresh(slots)), recordTrustedServerCreativeRequest: (auctionSlotId) => @@ -191,7 +197,9 @@ export class GptDiagnosticsApiController { requests: slot.requests.map((cycle) => ({ ...cycle, durations: { ...cycle.durations }, + requestedSlotSizes: cycle.requestedSlotSizes?.map((size) => [...size]), size: cycle.size ? [...cycle.size] : undefined, + observedSlotSize: cycle.observedSlotSize ? [...cycle.observedSlotSize] : undefined, adManager: cycle.adManager ? { ...cycle.adManager, diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts index 57fce3d85..408fcb1f9 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts @@ -106,6 +106,10 @@ function deliveryLabel(cycle: GptDiagnosticsRequestCycle): string | undefined { } } +function formatSizes(sizes: ReadonlyArray): string { + return sizes.map((size) => `${size[0]}×${size[1]}`).join(', '); +} + function badgeText(cycle: GptDiagnosticsRequestCycle): string { const firstLine: string[] = []; if (cycle.isEmpty === true) firstLine.push('Empty'); @@ -115,7 +119,13 @@ function badgeText(cycle: GptDiagnosticsRequestCycle): string { const delivery = deliveryLabel(cycle); if (delivery) firstLine.push(delivery); if (cycle.requestPath === 'competing') firstLine.push('Competing paths'); - if (cycle.size) firstLine.push(`${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.requestedSlotSizes) { + firstLine.push(`Requested ${formatSizes(cycle.requestedSlotSizes)}`); + } + if (cycle.size) firstLine.push(`GPT fill ${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.observedSlotSize) { + firstLine.push(`Outer box ${cycle.observedSlotSize[0]}×${cycle.observedSlotSize[1]}`); + } const timingLine: string[] = []; const response = formatMilliseconds(cycle.durations.requestToResponseMs); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts index 75bf97823..d7271710c 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts @@ -7,6 +7,7 @@ import { GptDiagnosticsBindingManager } from './binding'; import { GptDiagnosticsObserver } from './observer'; import type { GptObserverWindow } from './observer'; import { GptDiagnosticsOverlay } from './overlay'; +import { GptDiagnosticsSlotSizeObserver } from './slot_size_observer'; import { GptDiagnosticsStore } from './store'; interface GptDiagnosticsRuntime { @@ -44,6 +45,7 @@ export function installGptDiagnosticsRuntime( let bindings: GptDiagnosticsBindingManager | undefined; let badges: GptDiagnosticsBadgeManager | undefined; let overlay: GptDiagnosticsOverlay | undefined; + let slotSizeObserver: GptDiagnosticsSlotSizeObserver | undefined; let apiController: GptDiagnosticsApiController | undefined; try { @@ -59,6 +61,7 @@ export function installGptDiagnosticsRuntime( window: target, document: target.document, }); + slotSizeObserver = new GptDiagnosticsSlotSizeObserver(store, bindings, { window: target }); overlay = new GptDiagnosticsOverlay(store, bindings, { window: target, document: target.document, @@ -83,6 +86,7 @@ export function installGptDiagnosticsRuntime( apiController?.destroy(); overlay?.destroy(); badges?.destroy(); + slotSizeObserver?.destroy(); bindings?.destroy(); delete target.__tsjs_gpt_diagnostics_runtime; }, @@ -95,6 +99,7 @@ export function installGptDiagnosticsRuntime( apiController?.destroy(); overlay?.destroy(); badges?.destroy(); + slotSizeObserver?.destroy(); bindings?.destroy(); log.warn('gpt diagnostics: runtime installation failed', error); return undefined; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts index e99f1345b..1eeb4976e 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts @@ -277,7 +277,17 @@ function cycleFacts(cycle: GptDiagnosticsRequestCycle): string[] { if (cycle.loadAtMs !== undefined) facts.push('GPT slot onload observed'); if (cycle.viewableAtMs !== undefined) facts.push('GPT impressionViewable observed'); if (cycle.incompleteSequence) facts.push('Incomplete sequence'); - if (cycle.size) facts.push(`Rendered size ${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.requestedSlotSizes) { + facts.push( + `Requested slot sizes ${cycle.requestedSlotSizes + .map((size) => `${size[0]}×${size[1]}`) + .join(', ')}` + ); + } + if (cycle.size) facts.push(`GPT-reported fill size ${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.observedSlotSize) { + facts.push(`Observed outer slot box ${cycle.observedSlotSize[0]}×${cycle.observedSlotSize[1]}`); + } if (cycle.isBackfill !== undefined) facts.push(`Backfill ${cycle.isBackfill ? 'yes' : 'no'}`); if (cycle.slotContentChanged !== undefined) { facts.push(`Slot content changed ${cycle.slotContentChanged ? 'yes' : 'no'}`); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts new file mode 100644 index 000000000..ab49a451a --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts @@ -0,0 +1,146 @@ +import type { Size } from '../../core/types'; + +import type { GptDiagnosticsBindingManager } from './binding'; +import type { GptDiagnosticsStoreSnapshot } from './store'; + +interface SlotSizeStore { + snapshot(): GptDiagnosticsStoreSnapshot; + recordObservedSlotSize(runtimeSlotNumber: number, requestNumber: number, size: Size): void; + subscribe(listener: () => void): () => void; +} + +interface SlotSizeBindings { + get: GptDiagnosticsBindingManager['get']; + subscribe(listener: () => void): () => void; +} + +type SlotSizeWindow = Window & { + ResizeObserver?: typeof ResizeObserver; +}; + +interface SlotSizeObserverOptions { + window?: SlotSizeWindow; + scheduleFrame?: (callback: () => void) => void; +} + +interface ObservedCycle { + runtimeSlotNumber: number; + requestNumber: number; +} + +function defaultScheduleFrame(callback: () => void): void { + if (typeof requestAnimationFrame === 'function') { + requestAnimationFrame(() => callback()); + } else { + queueMicrotask(callback); + } +} + +function latestFilledCycle( + slot: GptDiagnosticsStoreSnapshot['slots'][number] +): ObservedCycle | undefined { + const cycle = slot.requests[slot.requests.length - 1]; + if (!cycle || cycle.isEmpty !== false || cycle.renderAtMs === undefined) return undefined; + return { runtimeSlotNumber: slot.runtimeSlotNumber, requestNumber: cycle.requestNumber }; +} + +/** + * Observes the outer CSS boxes of uniquely bound elements after filled GPT renders. + * + * Measurements remain separately labelled from GPT's reported creative size and + * are conditionally written with the runtime-slot and request-cycle identity that + * was current when the measurement was scheduled. + */ +export class GptDiagnosticsSlotSizeObserver { + private readonly store: SlotSizeStore; + private readonly bindings: SlotSizeBindings; + private readonly window: SlotSizeWindow; + private readonly scheduleFrame: (callback: () => void) => void; + private readonly unsubscribeStore: () => void; + private readonly unsubscribeBindings: () => void; + private resizeObserver?: ResizeObserver; + private refreshScheduled = false; + private destroyed = false; + + constructor( + store: SlotSizeStore, + bindings: SlotSizeBindings, + options: SlotSizeObserverOptions = {} + ) { + this.store = store; + this.bindings = bindings; + this.window = options.window ?? (window as unknown as SlotSizeWindow); + this.scheduleFrame = options.scheduleFrame ?? defaultScheduleFrame; + this.unsubscribeStore = this.store.subscribe(this.scheduleRefresh); + this.unsubscribeBindings = this.bindings.subscribe(this.scheduleRefresh); + this.refresh(); + } + + destroy(): void { + if (this.destroyed) return; + this.destroyed = true; + this.unsubscribeStore(); + this.unsubscribeBindings(); + this.resizeObserver?.disconnect(); + } + + private readonly scheduleRefresh = (): void => { + if (this.destroyed || this.refreshScheduled) return; + this.refreshScheduled = true; + this.scheduleFrame(() => { + this.refreshScheduled = false; + this.refresh(); + }); + }; + + private refresh(): void { + if (this.destroyed) return; + this.resizeObserver?.disconnect(); + const observations = new Map(); + const ResizeObserverConstructor = this.window.ResizeObserver; + if (typeof ResizeObserverConstructor === 'function') { + this.resizeObserver = new ResizeObserverConstructor((entries) => { + for (const entry of entries) { + const element = entry.target; + if (!(element instanceof this.window.HTMLElement)) continue; + const cycle = observations.get(element); + if (cycle) this.scheduleMeasure(element, cycle); + } + }); + } + + for (const slot of this.store.snapshot().slots) { + const cycle = latestFilledCycle(slot); + const binding = this.bindings.get(slot.runtimeSlotNumber); + if (!cycle || binding.binding.status !== 'bound' || !binding.element?.isConnected) continue; + observations.set(binding.element, cycle); + this.resizeObserver?.observe(binding.element); + this.scheduleMeasure(binding.element, cycle); + } + } + + private scheduleMeasure(element: HTMLElement, cycle: ObservedCycle): void { + this.scheduleFrame(() => this.measure(element, cycle)); + } + + private measure(element: HTMLElement, cycle: ObservedCycle): void { + const binding = this.bindings.get(cycle.runtimeSlotNumber); + if (binding.binding.status !== 'bound' || binding.element !== element || !element.isConnected) { + return; + } + + const rectangle = element.getBoundingClientRect(); + if ( + !Number.isFinite(rectangle.width) || + !Number.isFinite(rectangle.height) || + rectangle.width < 0 || + rectangle.height < 0 + ) { + return; + } + this.store.recordObservedSlotSize(cycle.runtimeSlotNumber, cycle.requestNumber, [ + rectangle.width, + rectangle.height, + ]); + } +} diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts index 0324a56cc..03d887aa0 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts @@ -21,6 +21,7 @@ export const MAX_DIAGNOSTIC_SLOTS = 64; export const MAX_REQUEST_CYCLES_PER_SLOT = 10; export const MAX_CALLBACK_ISSUES = 128; export const MAX_TRUSTED_SERVER_ASSOCIATIONS = 64; +export const MAX_REQUESTED_SLOT_SIZES = 16; export const CREATIVE_ATTEMPT_WINDOW_MS = 30_000; export const MAX_CREATIVE_ATTEMPTS = 128; export const MAX_ATTRIBUTION_ISSUES = 128; @@ -106,6 +107,7 @@ interface PendingSourceEvidence { observedAtMs: number; trustedServerOpportunity?: GptDiagnosticsTrustedServerOpportunity; trustedServerAuctionId?: string; + requestedSlotSizes?: ReadonlyArray; } interface PendingRequestIntent { @@ -205,6 +207,29 @@ function normalizedAuctionId(value: unknown): string | undefined { return new TextEncoder().encode(trimmed).length <= 256 ? trimmed : undefined; } +function normalizedRequestedSlotSizes(value: unknown): ReadonlyArray | undefined { + if (!Array.isArray(value)) return undefined; + + const requestedSlotSizes: Size[] = []; + for (const candidate of value.slice(0, MAX_REQUESTED_SLOT_SIZES)) { + if ( + !Array.isArray(candidate) || + candidate.length !== 2 || + typeof candidate[0] !== 'number' || + typeof candidate[1] !== 'number' || + !Number.isFinite(candidate[0]) || + !Number.isFinite(candidate[1]) || + candidate[0] <= 0 || + candidate[1] <= 0 + ) { + continue; + } + requestedSlotSizes.push(Object.freeze([candidate[0], candidate[1]] as [number, number])); + } + + return requestedSlotSizes.length > 0 ? Object.freeze(requestedSlotSizes) : undefined; +} + function responseClass(cycle: MutableRequestCycle): GptDiagnosticsResponseClass | undefined { if (cycle.renderAtMs === undefined) return undefined; if (cycle.isEmpty === true) return 'empty'; @@ -248,7 +273,9 @@ function copyCycle(cycle: MutableRequestCycle, nowMs: number): GptDiagnosticsReq return { ...cycle, durations: derivedDurations(cycle), + requestedSlotSizes: cycle.requestedSlotSizes?.map((size) => [...size] as Size), size: cycle.size ? ([...cycle.size] as Size) : undefined, + observedSlotSize: cycle.observedSlotSize ? ([...cycle.observedSlotSize] as Size) : undefined, adManager: cycle.adManager ? { ...cycle.adManager, @@ -309,7 +336,8 @@ export class GptDiagnosticsStore { slot: GptDiagnosticsSlotLike, auctionSlotId: string, opportunity: GptDiagnosticsTrustedServerOpportunity, - trustedServerAuctionId?: string + trustedServerAuctionId?: string, + requestedSlotSizes?: ReadonlyArray ): void { if ( !isSlotObject(slot) || @@ -331,6 +359,7 @@ export class GptDiagnosticsStore { this.recordRequestIntentSource(slot, 'trusted_server_direct', { trustedServerOpportunity: opportunity, trustedServerAuctionId: normalizedAuctionId(trustedServerAuctionId), + requestedSlotSizes: normalizedRequestedSlotSizes(requestedSlotSizes), }); } @@ -578,6 +607,9 @@ export class GptDiagnosticsStore { ...(trustedServerEvidence?.trustedServerAuctionId !== undefined ? { trustedServerAuctionId: trustedServerEvidence.trustedServerAuctionId } : {}), + ...(trustedServerEvidence?.requestedSlotSizes !== undefined + ? { requestedSlotSizes: trustedServerEvidence.requestedSlotSizes } + : {}), ...(trustedServerEvidence ? { opportunityToRequestMs: validDuration(trustedServerEvidence.observedAtMs, timestampMs), @@ -666,6 +698,45 @@ export class GptDiagnosticsStore { ); } + /** + * Retain an outer CSS box only when this exact slot and request cycle still + * identify a filled render. Async DOM measurements use this guard so a prior + * render cannot alter a later refresh cycle. + */ + recordObservedSlotSize(runtimeSlotNumber: number, requestNumber: number, size: Size): void { + if ( + !Number.isSafeInteger(requestNumber) || + requestNumber <= 0 || + !Number.isFinite(size[0]) || + !Number.isFinite(size[1]) || + size[0] < 0 || + size[1] < 0 + ) { + return; + } + + const record = this.slots.get(runtimeSlotNumber); + const cycle = record?.requests.find((candidate) => candidate.requestNumber === requestNumber); + if ( + !cycle || + record.requests[record.requests.length - 1] !== cycle || + cycle.isEmpty !== false || + cycle.renderAtMs === undefined + ) { + return; + } + + const observedSlotSize: Size = [size[0], size[1]]; + if ( + cycle.observedSlotSize?.[0] === observedSlotSize[0] && + cycle.observedSlotSize[1] === observedSlotSize[1] + ) { + return; + } + cycle.observedSlotSize = observedSlotSize; + this.notify(); + } + recordSlotOnload(slot: GptDiagnosticsSlotLike): void { const timestampMs = this.timestamp(); this.matchCycle( @@ -861,7 +932,10 @@ export class GptDiagnosticsStore { private recordRequestIntentSource( slot: object, source: RequestIntentSource, - facts: Pick = {} + facts: Pick< + PendingSourceEvidence, + 'trustedServerOpportunity' | 'trustedServerAuctionId' | 'requestedSlotSizes' + > = {} ): void { const observedAtMs = this.now(); let intent = this.pendingRequestIntents.get(slot); diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index dc17c9e87..1f8e032ba 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { AdUnit } from '../../src/core/types'; +import { APS_RENDERING_MODE_META_NAME } from '../../src/integrations/aps/render'; import envelope from '../fixtures/aps-renderer-v1.json'; async function flushRequestAds(): Promise { @@ -137,6 +138,57 @@ describe('request.requestAds', () => { expect(document.querySelector('#slot1 span')).toBeNull(); }); + it('contract test: dispatches a direct APS bid to the native publisher hook without an iframe', async () => { + const apsBid = envelope.seatbid[0].bid[0]; + const renderer = { + type: 'aps' as const, + version: 1 as const, + accountId: 'example-account-id', + bidId: apsBid.id, + tagType: apsBid.ext.tagtype as 'iframe', + creativeUrl: apsBid.ext.creativeurl, + aaxResponse: btoa(JSON.stringify(envelope)), + width: apsBid.w, + height: apsBid.h, + }; + const render = vi.fn().mockResolvedValue({ accepted: true }); + window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; + const marker = document.createElement('meta'); + marker.name = APS_RENDERING_MODE_META_NAME; + marker.content = 'publisher_native'; + document.head.appendChild(marker); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({ + seatbid: [ + { + seat: 'aps', + bid: [{ impid: 'slot1', ext: { trusted_server: { renderer } } }], + }, + ], + }), + }); + + try { + const { addAdUnits } = await import('../../src/core/registry'); + const { requestAds } = await import('../../src/core/request'); + document.body.innerHTML = '
existing
'; + addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } }); + + requestAds(); + await flushRequestAds(); + await Promise.resolve(); + + expect(render).toHaveBeenCalledWith({ version: 1, slotId: 'slot1', renderer }); + expect(document.querySelector('#slot1 iframe')).toBeNull(); + expect(document.querySelector('#slot1 span')).not.toBeNull(); + } finally { + marker.remove(); + } + }); + it('does not mutate the slot for an invalid APS descriptor', async () => { globalThis.fetch = vi.fn().mockResolvedValue({ ok: true, diff --git a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts index eae60c90e..627ae1268 100644 --- a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts @@ -4,11 +4,14 @@ import envelope from '../../fixtures/aps-renderer-v1.json'; import type { ApsRendererV1 } from '../../../src/core/types'; import { log } from '../../../src/core/log'; import { + APS_NATIVE_RENDERER_ACK_TIMEOUT_MS, APS_RENDERER_PATH, APS_RENDERER_SANDBOX, + APS_RENDERING_MODE_META_NAME, APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, apsRendererUrl, + dispatchApsRendering, getApsPrebidRenderer, parseApsRendererDescriptor, registerApsPrebidRenderer, @@ -16,6 +19,19 @@ import { validateApsRenderer, } from '../../../src/integrations/aps/render'; +function enablePublisherNativeMode(): void { + const marker = document.createElement('meta'); + marker.name = APS_RENDERING_MODE_META_NAME; + marker.content = 'publisher_native'; + document.head.appendChild(marker); +} + +function disablePublisherNativeMode(): void { + document.head + .querySelectorAll(`meta[name="${APS_RENDERING_MODE_META_NAME}"]`) + .forEach((marker) => marker.remove()); +} + function encodeBytes(bytes: Uint8Array): string { let binary = ''; for (const byte of bytes) binary += String.fromCharCode(byte); @@ -261,6 +277,256 @@ describe('Prebid APS renderer registry', () => { }); }); +describe('publisher-native APS hook contract tests', () => { + beforeEach(() => { + document.body.innerHTML = '
existing
'; + enablePublisherNativeMode(); + }); + + afterEach(() => { + disablePublisherNativeMode(); + delete window.tsjs; + vi.restoreAllMocks(); + document.body.innerHTML = ''; + }); + + it('contract test: sends the exact frozen descriptor to an accepting publisher hook without an iframe', async () => { + const render = vi.fn().mockResolvedValue({ accepted: true }); + window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; + const unrelatedMarker = document.createElement('meta'); + unrelatedMarker.name = APS_RENDERING_MODE_META_NAME; + unrelatedMarker.content = 'trusted_server'; + document.head.appendChild(unrelatedMarker); + + const accepted = await dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => { + throw new Error('trusted renderer must not run'); + }, + }); + + expect(accepted).toBe(true); + expect(render).toHaveBeenCalledTimes(1); + expect(render).toHaveBeenCalledWith({ + version: 1, + slotId: 'fictional-slot', + renderer: descriptor(), + }); + const payload = render.mock.calls[0][0]; + expect(Object.keys(payload).sort()).toEqual(['renderer', 'slotId', 'version']); + expect(Object.isFrozen(payload.renderer)).toBe(true); + expect(document.querySelector('iframe')).toBeNull(); + }); + + it('contract test: declines missing, throwing, rejecting, and malformed hooks without fallback', async () => { + const trustedServer = vi.fn(() => true); + await expect( + dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) + ).resolves.toBe(false); + + window.tsjs = { + apsNativeRenderer: { + render: () => { + throw new Error('fictional'); + }, + }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) + ).resolves.toBe(false); + + window.tsjs = { + apsNativeRenderer: { render: () => Promise.reject(new Error('fictional')) }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) + ).resolves.toBe(false); + + window.tsjs = { + apsNativeRenderer: { render: () => ({ accepted: false }) }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) + ).resolves.toBe(false); + + window.tsjs = { + apsNativeRenderer: { render: () => ({ accepted: 'yes' }) }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ slotId: 'fictional-slot', renderer: descriptor(), trustedServer }) + ).resolves.toBe(false); + + expect(trustedServer).not.toHaveBeenCalled(); + expect(document.querySelector('iframe')).toBeNull(); + }); + + it('contract test: ignores a stale acknowledgement after a replacement dispatch', async () => { + let resolveFirst: ((value: { accepted: boolean }) => void) | undefined; + const render = vi + .fn() + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFirst = resolve; + }) + ) + .mockResolvedValueOnce({ accepted: true }); + window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; + + const first = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + const second = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + resolveFirst!({ accepted: true }); + + await expect(first).resolves.toBe(false); + await expect(second).resolves.toBe(true); + }); + + it('contract test: a missing hook supersedes an older pending dispatch', async () => { + let resolveFirst: ((value: { accepted: boolean }) => void) | undefined; + const render = vi.fn( + () => + new Promise<{ accepted: boolean }>((resolve) => { + resolveFirst = resolve; + }) + ); + window.tsjs = { apsNativeRenderer: { render } } as typeof window.tsjs; + + const first = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + delete window.tsjs!.apsNativeRenderer; + const second = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + resolveFirst!({ accepted: true }); + + await expect(second).resolves.toBe(false); + await expect(first).resolves.toBe(false); + }); + + it('contract test: a trusted-server dispatch supersedes an older native dispatch', async () => { + let resolveFirst: ((value: { accepted: boolean }) => void) | undefined; + window.tsjs = { + apsNativeRenderer: { + render: () => + new Promise<{ accepted: boolean }>((resolve) => { + resolveFirst = resolve; + }), + }, + } as typeof window.tsjs; + const first = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + + disablePublisherNativeMode(); + const trustedServer = vi.fn(() => true); + const second = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer, + }); + resolveFirst!({ accepted: true }); + + expect(second).toBe(true); + expect(trustedServer).toHaveBeenCalledOnce(); + await expect(first).resolves.toBe(false); + }); + + it('contract test: contains throwing hook and acknowledgement accessors', async () => { + const tsjs = {} as NonNullable; + Object.defineProperty(tsjs, 'apsNativeRenderer', { + get: () => { + throw new Error('fictional hook lookup failure'); + }, + }); + window.tsjs = tsjs; + + await expect( + dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }) + ).resolves.toBe(false); + + window.tsjs = { + apsNativeRenderer: { + render: () => + new Proxy( + { accepted: true }, + { + ownKeys: () => { + throw new Error('fictional acknowledgement inspection failure'); + }, + } + ), + }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }) + ).resolves.toBe(false); + }); + + it('contract test: times out a hook and ignores its late acknowledgement', async () => { + vi.useFakeTimers(); + try { + let resolveHook: ((value: { accepted: boolean }) => void) | undefined; + window.tsjs = { + apsNativeRenderer: { + render: () => + new Promise<{ accepted: boolean }>((resolve) => { + resolveHook = resolve; + }), + }, + } as typeof window.tsjs; + + const result = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + await vi.advanceTimersByTimeAsync(APS_NATIVE_RENDERER_ACK_TIMEOUT_MS); + + await expect(result).resolves.toBe(false); + expect(vi.getTimerCount()).toBe(0); + resolveHook!({ accepted: true }); + await Promise.resolve(); + + window.tsjs = { + apsNativeRenderer: { render: () => ({ accepted: true }) }, + } as typeof window.tsjs; + await expect( + dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }) + ).resolves.toBe(true); + } finally { + vi.useRealTimers(); + } + }); +}); + describe('direct APS rendering', () => { beforeEach(() => { document.body.innerHTML = '
existing
'; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 179a810d5..6eb41e904 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -7,6 +7,15 @@ import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vites import envelope from '../../fixtures/aps-renderer-v1.json'; import type { AuctionBidData, TsjsApi } from '../../../src/core/types'; +import { APS_RENDERING_MODE_META_NAME } from '../../../src/integrations/aps/render'; + +function enablePublisherNativeMode(): HTMLMetaElement { + const marker = document.createElement('meta'); + marker.name = APS_RENDERING_MODE_META_NAME; + marker.content = 'publisher_native'; + document.head.appendChild(marker); + return marker; +} function apsRenderer() { const bid = envelope.seatbid[0].bid[0]; @@ -206,7 +215,8 @@ describe('installTsAdInit', () => { function configureOpportunityDiagnostics( bid: AuctionBidData | undefined, - recordTrustedServerOpportunity: ReturnType + recordTrustedServerOpportunity: ReturnType, + formats: Array<[number, number]> = [[300, 250]] ) { const mockSlot = { addService: vi.fn().mockReturnThis(), @@ -232,7 +242,7 @@ describe('installTsAdInit', () => { id: 'atf_sidebar_ad', gam_unit_path: '/123/atf', div_id: 'div-atf-sidebar', - formats: [[300, 250]], + formats, targeting: {}, }, ], @@ -293,7 +303,9 @@ describe('installTsAdInit', () => { expect(recordTrustedServerOpportunity).toHaveBeenCalledWith( mockSlot, 'atf_sidebar_ad', - expectedOpportunity + expectedOpportunity, + undefined, + [[300, 250]] ); } ); @@ -318,7 +330,34 @@ describe('installTsAdInit', () => { mockSlot, 'atf_sidebar_ad', 'unrenderable_candidate', - 'auction-123' + 'auction-123', + [[300, 250]] + ); + }); + + it('captures every configured Trusted Server format when associating a GPT slot', async () => { + const recordTrustedServerOpportunity = vi.fn(); + const formats: Array<[number, number]> = [ + [300, 250], + [728, 90], + [320, 50], + ]; + const { mockSlot } = configureOpportunityDiagnostics( + undefined, + recordTrustedServerOpportunity, + formats + ); + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(recordTrustedServerOpportunity).toHaveBeenCalledWith( + mockSlot, + 'atf_sidebar_ad', + 'no_candidate', + undefined, + formats ); }); @@ -334,7 +373,9 @@ describe('installTsAdInit', () => { expect(recordTrustedServerOpportunity).toHaveBeenCalledWith( mockSlot, 'atf_sidebar_ad', - 'no_candidate' + 'no_candidate', + undefined, + [[300, 250]] ); }); @@ -1874,7 +1915,7 @@ describe('installTsAdInit', () => { setTargeting: vi.fn().mockReturnThis(), clearTargeting, getSlotElementId: vi.fn().mockReturnValue('div-old-route'), - getTargeting: vi.fn().mockReturnValue([]), + getTargeting: vi.fn((key: string) => (key === 'ts' ? ['publisher-value'] : [])), }; const mockPubads = { enableSingleRequest: vi.fn(), @@ -1908,6 +1949,7 @@ describe('installTsAdInit', () => { expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); expect(clearTargeting).toHaveBeenCalledWith('pos'); + expect(clearTargeting).not.toHaveBeenCalledWith('ts'); expect(mockPubads.refresh).not.toHaveBeenCalled(); expect((window as TestWindow).tsjs!.divToSlotId).toEqual({}); expect((window as TestWindow).tsjs!.prevSlotTargetingKeys).toEqual({}); @@ -3197,6 +3239,79 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('contract test: delegates a server APS owner to the native hook without a Universal Creative response', async () => { + const renderer = apsRenderer(); + const render = vi.fn().mockResolvedValue({ accepted: true }); + (window as TestWindow).tsjs.bids.homepage_header = { + hb_adid: renderer.bidId, + renderer, + }; + (window as TestWindow).tsjs.apsNativeRenderer = { render }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: renderer.bidId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + bridgeListener(request); + await Promise.resolve(); + await Promise.resolve(); + + expect(render).toHaveBeenCalledOnce(); + expect(render).toHaveBeenCalledWith({ + version: 1, + slotId: 'homepage_header', + renderer, + }); + expect(portMessages).toEqual([]); + expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); + } finally { + marker.remove(); + } + }); + + it('contract test: declines a server APS owner without a Universal Creative response or fallback', async () => { + const renderer = apsRenderer(); + const render = vi.fn().mockResolvedValue({ accepted: false }); + (window as TestWindow).tsjs.bids.homepage_header = { + hb_adid: renderer.bidId, + renderer, + }; + (window as TestWindow).tsjs.apsNativeRenderer = { render }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: renderer.bidId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + await Promise.resolve(); + await Promise.resolve(); + bridgeListener(request); + + expect(render).toHaveBeenCalledOnce(); + expect(portMessages).toEqual([]); + expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); + } finally { + marker.remove(); + } + }); + it('serves a registered Prebid APS renderer when its generated ad ID differs from the APS bid ID', async () => { const renderer = apsRenderer(); const prebidAdId = 'prebid-generated-ad-id'; @@ -3252,6 +3367,92 @@ describe('installTsRenderBridge', () => { foreignIframe.remove(); }); + it('contract test: declines a registered APS capability without a Universal Creative response or markUsed', async () => { + const renderer = apsRenderer(); + const prebidAdId = 'native-prebid-decline-ad-id'; + const markUsed = vi.fn(); + const render = vi.fn().mockResolvedValue({ accepted: false }); + (window as TestWindow).tsjs.apsPrebidRenderers = { + [prebidAdId]: { + adUnitCode: 'div-header', + renderer, + registeredAt: Date.now(), + expiresAt: Date.now() + 60_000, + markUsed, + }, + }; + (window as TestWindow).tsjs.apsNativeRenderer = { render }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: prebidAdId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + await Promise.resolve(); + await Promise.resolve(); + bridgeListener(request); + + expect(render).toHaveBeenCalledOnce(); + expect(markUsed).not.toHaveBeenCalled(); + expect(portMessages).toEqual([]); + expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); + expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); + } finally { + marker.remove(); + } + }); + + it('contract test: consumes a registered APS capability and marks it used only after native acceptance', async () => { + const renderer = apsRenderer(); + const prebidAdId = 'native-prebid-ad-id'; + const markUsed = vi.fn(); + const render = vi.fn().mockResolvedValue({ accepted: true }); + (window as TestWindow).tsjs.apsPrebidRenderers = { + [prebidAdId]: { + adUnitCode: 'div-header', + renderer, + registeredAt: Date.now(), + expiresAt: Date.now() + 60_000, + markUsed, + }, + }; + (window as TestWindow).tsjs.apsNativeRenderer = { render }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: prebidAdId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + expect(markUsed).not.toHaveBeenCalled(); + await Promise.resolve(); + await Promise.resolve(); + bridgeListener(request); + + expect(render).toHaveBeenCalledOnce(); + expect(markUsed).toHaveBeenCalledOnce(); + expect(portMessages).toEqual([]); + expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); + } finally { + marker.remove(); + } + }); + it('still serves the APS renderer when markUsed throws', async () => { const renderer = apsRenderer(); const prebidAdId = 'throwing-mark-used-ad-id'; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index d3e1d7099..74c704cdc 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -33,6 +33,8 @@ interface MockGoogleTag { pubads: () => unknown; enableServices: () => void; display: (divId: string) => void; + getConfig?: (key: string) => Record; + setConfig?: (config: Record) => void; } // `tsjs` is declared globally as the full `TsjsApi`; `Omit` drops it from @@ -40,8 +42,25 @@ interface MockGoogleTag { type TestWindow = Omit & { googletag?: MockGoogleTag; tsjs?: Partial; + __tsjs_gam_attribution_enabled?: boolean; }; +function makeGoogleTag(overrides: Partial = {}): MockGoogleTag { + const pubads = { + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + + return { + cmd: [], + defineSlot: vi.fn(), + pubads: vi.fn(() => pubads), + enableServices: vi.fn(), + display: vi.fn(), + ...overrides, + }; +} + function runBootstrap(): void { // Evaluate in the jsdom global scope, exactly as an inline '); + clonedDocument.close(); + executingScript = clonedDocument.querySelector('script'); + const queue: Array<() => void> = []; + const setConfig = vi.fn(); + win.googletag = makeGoogleTag({ cmd: queue, setConfig }); + + await importFreshGptBundle(); + queue[0](); + + expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }); + }); + + it.each(['missing', 'throwing'])( + 'keeps module installation working with %s setConfig', + async (setConfigMode) => { + const queue: Array<() => void> = []; + const setConfig = + setConfigMode === 'throwing' + ? vi.fn(() => { + throw new Error('publisher setConfig failed'); + }) + : undefined; + win.googletag = makeGoogleTag({ cmd: queue, setConfig }); + win.__tsjs_slim_prebid_url = 'https://cdn.example.com/slim-prebid.js'; + executingScript = attributedScript(); + + await importFreshGptBundle(); + + expect(() => [...queue].forEach((command) => command())).not.toThrow(); + expect(typeof win.tsjs?.adInit).toBe('function'); + expect(typeof win.tsjs?.scheduleInitialAdInit).toBe('function'); + expect(win.tsjs?.spaHookInstalled).toBe(true); + expect(addEventListenerSpy).toHaveBeenCalledWith('popstate', expect.any(Function)); + expect(addEventListenerSpy).toHaveBeenCalledWith('load', expect.any(Function)); + expect(addEventListenerSpy).toHaveBeenCalledWith('message', expect.any(Function)); + if (setConfig) { + expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }); + } + } + ); + + it('preserves GPT-enabled shim behavior without queuing attribution when unmarked', async () => { + const queue: Array<() => void> = []; + const setConfig = vi.fn(); + const tag = makeGoogleTag({ cmd: queue, setConfig }); + win.googletag = tag; + win.__tsjs_gpt_enabled = true; + executingScript = document.createElement('script'); + + await importFreshGptBundle(); + [...queue].forEach((command) => command()); + const guard = await importGuardModule(); + + expect(guard.isGuardInstalled()).toBe(true); + expect(win.googletag).toBe(tag); + expect(win.googletag!.cmd).toBe(queue); + expect(setConfig).not.toHaveBeenCalled(); + expect(typeof win.tsjs?.adInit).toBe('function'); + }); +}); + describe('GPT debug ADM iframe hardening', () => { it('sandbox token list omits allow-same-origin', async () => { const mod = await import('../../../src/integrations/gpt/index'); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts index 999c60c55..830bac217 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts @@ -310,6 +310,69 @@ describe('scheduleInitialAdInit', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('applies the SSR slot definitions on the initial document', async () => { + // Under a shared-template mode the head script emits no `tsjs.adSlots`, so the + // `` seam is the only source of slot definitions. They must arrive, or + // `adInit()` iterates an empty list and the page defines no TS slots at all. + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + const ssrSlot = { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]] as Array<[number, number]>, + }; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + + expect(ts.adSlots).toEqual([ssrSlot]); + expect(ts.bids).toEqual({ ssr_slot: { hb_pb: '1.00' } }); + }); + + it('drops the SSR slot definitions when a navigation has already committed', async () => { + // The guard covered the bids and the adInit call, but the shared-template seam + // assigned `tsjs.adSlots` on the line *before* calling the scheduler — outside the + // guard entirely. A navigation that committed while the SSR document was still + // streaming therefore kept its own bids and silently lost its slots to the stale + // SSR payload, and the next `adInit()` for that route defined the wrong slots. + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/b'); + await flushAsync(); + expect(ts.navGeneration).toBe(1); + const liveSlot = { + id: 'live_slot', + gam_unit_path: '/123/live', + div_id: 'div-live', + formats: [[300, 250]] as Array<[number, number]>, + }; + ts.adSlots = [liveSlot]; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ + { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]], + }, + ]); + + expect(ts.adSlots).toEqual([liveSlot]); + + window.dispatchEvent(new Event('load')); + flushFrame(); + flushFrame(); + expect(adInit).not.toHaveBeenCalled(); + }); + it('cancels queued GPT work when a navigation commits before the command queue drains', async () => { // adInit() only queues its slot work on googletag.cmd, which drains when // GPT itself loads — possibly long after the generation check that diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 7127efcb4..314348fa8 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -216,9 +216,9 @@ describe('installSpaAuctionHook', () => { }); it('skips adInit on an empty page-bids response with no prior TS state', async () => { - // A gated page-bids response (auction kill switch or consent denial) returns - // no slots. With no prior TS state to sweep, the hook must not call adInit() - // so a consent-denied navigation cannot activate the publisher's GPT setup. + // A gated page-bids response (template switch, auction gate, or consent + // denial) returns no slots. With no prior TS state to sweep, the hook must + // not call adInit() so a gated navigation cannot activate publisher GPT. fetchStub.mockResolvedValue({ ok: true, json: async () => ({ slots: [], bids: {} }), diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts index 2e2ae2d2b..2e3a63b4a 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts @@ -124,7 +124,9 @@ describe('GptDiagnosticsApiController', () => { expect(store.recordTrustedServerOpportunity).toHaveBeenCalledWith( slot, 'auction-slot-example', - 'renderable_candidate' + 'renderable_candidate', + undefined, + undefined ); expect(store.recordPrebidRefresh).toHaveBeenCalledTimes(1); expect(store.recordPrebidRefresh).toHaveBeenCalledWith(slots); @@ -156,7 +158,8 @@ describe('GptDiagnosticsApiController', () => { slot, 'auction-slot-example', 'renderable_candidate', - 'auction-123' + 'auction-123', + undefined ); }); @@ -214,6 +217,10 @@ describe('GptDiagnosticsApiController', () => { requestNumber: 1, durations: {}, incompleteSequence: false, + requestedSlotSizes: [ + [300, 250], + [728, 90], + ], adManager: { yieldGroupIds: [10], companyIds: [20], @@ -253,6 +260,14 @@ describe('GptDiagnosticsApiController', () => { expect(snapshot.attributionIssues).toEqual(source.attributionIssues); expect(snapshot.attributionIssues).not.toBe(source.attributionIssues); expect(snapshot.attributionIssues?.[0]).not.toBe(source.attributionIssues[0]); + expect(cycle?.requestedSlotSizes).toEqual([ + [300, 250], + [728, 90], + ]); + expect(cycle?.requestedSlotSizes).not.toBe(source.slots[0]?.requests[0]?.requestedSlotSizes); + expect(cycle?.requestedSlotSizes?.[0]).not.toBe( + source.slots[0]?.requests[0]?.requestedSlotSizes?.[0] + ); expect(cycle?.trustedServerCreativeFailures).toEqual(['cache_fetch_failed']); expect(cycle?.trustedServerCreativeFailures).not.toBe( source.slots[0]?.requests[0]?.trustedServerCreativeFailures @@ -348,10 +363,12 @@ describe('GptDiagnosticsApiController', () => { 'incompleteSequence', 'isBackfill', 'isEmpty', + 'observedSlotSize', 'renderAtMs', 'requestNumber', 'requestPath', 'requestedAtMs', + 'requestedSlotSizes', 'responseAtMs', 'responseClass', 'size', @@ -428,6 +445,10 @@ describe('GptDiagnosticsApiController', () => { requestNumber: 1, durations: { requestToResponseMs: 10 }, incompleteSequence: false, + requestedSlotSizes: [ + [300, 250], + [728, 90], + ], adManager: { yieldGroupIds: [10], companyIds: [20] }, trustedServerCreativeFailures: ['cache_fetch_failed' as const], }, @@ -463,6 +484,9 @@ describe('GptDiagnosticsApiController', () => { controller.api.subscribe((snapshot) => { const cycle = snapshot.slots[0]!.requests[0]!; cycle.durations.requestToResponseMs = 999; + const requestedSlotSizes = cycle.requestedSlotSizes as unknown as Array<[number, number]>; + requestedSlotSizes[0]![0] = 1; + requestedSlotSizes.push([970, 250]); cycle.adManager!.yieldGroupIds!.push(99); cycle.trustedServerCreativeFailures!.push('response_post_failed'); snapshot.attributionIssues?.push({ @@ -484,6 +508,10 @@ describe('GptDiagnosticsApiController', () => { expect(observedSnapshot?.capturedAt).toBe('2026-08-10T00:00:00.000Z'); const observedCycle = observedSnapshot?.slots[0]?.requests[0]; expect(observedCycle?.durations.requestToResponseMs).toBe(10); + expect(observedCycle?.requestedSlotSizes).toEqual([ + [300, 250], + [728, 90], + ]); expect(observedCycle?.adManager?.yieldGroupIds).toEqual([10]); expect(observedCycle?.trustedServerCreativeFailures).toEqual(['cache_fetch_failed']); expect(observedSnapshot?.attributionIssues).toHaveLength(1); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts index 675e4f442..7ac981b6d 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts @@ -252,7 +252,12 @@ describe('GptDiagnosticsBadgeManager', () => { renderAtMs: 318, viewableAtMs: 1318, isEmpty: false, + requestedSlotSizes: [ + [728, 90], + [970, 250], + ], size: [728, 90], + observedSlotSize: [980, 270], incompleteSequence: false, durations: { requestToResponseMs: 276, @@ -260,7 +265,9 @@ describe('GptDiagnosticsBadgeManager', () => { renderToViewableMs: 1000, }, }) - ).toBe('Filled · 728×90\nResponse 276 ms · Render 42 ms\nViewable after 1 s'); + ).toBe( + 'Filled · Requested 728×90, 970×250 · GPT fill 728×90 · Outer box 980×270\nResponse 276 ms · Render 42 ms\nViewable after 1 s' + ); expect( gptDiagnosticsBadgeTextForTest({ requestNumber: 1, diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts index b89a8f507..9c9765ed1 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts @@ -405,6 +405,16 @@ describe('GptDiagnosticsOverlay', () => { const element = document.createElement('div'); element.id = 'filled-slot'; document.body.append(element); + store.recordTrustedServerOpportunity( + filledSlot, + 'filled-slot-auction', + 'renderable_candidate', + undefined, + [ + [300, 250], + [728, 90], + ] + ); store.recordSlotRequested(filledSlot); now = 20; store.recordSlotResponseReceived(filledSlot); @@ -414,6 +424,7 @@ describe('GptDiagnosticsOverlay', () => { size: [300, 250], isBackfill: true, }); + store.recordObservedSlotSize(1, 1, [320, 270]); now = 30; store.recordSlotOnload(filledSlot); now = 35; @@ -457,7 +468,9 @@ describe('GptDiagnosticsOverlay', () => { expect(root!.textContent).toContain('/example/site/filled-slot'); expect(root!.textContent).toContain('Empty'); expect(root!.textContent).toContain('Previous requests (1)'); - expect(root!.textContent).toContain('Rendered size 300×250'); + expect(root!.textContent).toContain('Requested slot sizes 300×250, 728×90'); + expect(root!.textContent).toContain('GPT-reported fill size 300×250'); + expect(root!.textContent).toContain('Observed outer slot box 320×270'); expect(root!.textContent).toContain('Backfill yes'); expect(root!.textContent).toContain('GPT slot onload observed'); expect(root!.textContent).toContain('GPT impressionViewable observed'); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts new file mode 100644 index 000000000..86ad532c7 --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts @@ -0,0 +1,158 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import type { GptDiagnosticsRequestCycle } from '../../../src/core/types'; +import { GptDiagnosticsSlotSizeObserver } from '../../../src/integrations/gpt_diagnostics/slot_size_observer'; +import type { GptDiagnosticsStoreSnapshot } from '../../../src/integrations/gpt_diagnostics/store'; + +class ResizeObserverMock { + static instances: ResizeObserverMock[] = []; + readonly observe = vi.fn(); + readonly disconnect = vi.fn(); + + constructor(readonly callback: ResizeObserverCallback) { + ResizeObserverMock.instances.push(this); + } + + emit(element: Element): void { + this.callback([{ target: element } as ResizeObserverEntry], this as unknown as ResizeObserver); + } +} + +function cycle(requestNumber: number, isEmpty: boolean | undefined): GptDiagnosticsRequestCycle { + return { + requestNumber, + isEmpty, + renderAtMs: 1, + durations: {}, + incompleteSequence: false, + }; +} + +function snapshot(requests: GptDiagnosticsRequestCycle[]): GptDiagnosticsStoreSnapshot { + return { + gptObserved: true, + slots: [ + { + runtimeSlotNumber: 1, + slotElementId: 'ad-slot-example', + requests, + }, + ], + callbackIssues: [], + attributionIssues: [], + coverage: { + slotRequested: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotResponseReceived: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotRenderEnded: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotOnload: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + impressionViewable: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotVisibilityChanged: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + }, + metadata: { + droppedCallbacks: 0, + droppedAttributionIssues: 0, + evictedSlots: 0, + evictedRequestCycles: 0, + }, + }; +} + +describe('GptDiagnosticsSlotSizeObserver', () => { + afterEach(() => { + ResizeObserverMock.instances = []; + document.body.replaceChildren(); + }); + + it('keeps GPT 1×1 distinct from the observed outer box and updates it on resize', () => { + const element = document.createElement('div'); + document.body.append(element); + const getBoundingClientRect = vi.spyOn(element, 'getBoundingClientRect'); + getBoundingClientRect.mockReturnValue({ width: 728, height: 90 } as DOMRect); + const requests = [cycle(1, false)]; + requests[0].size = [1, 1]; + const store = { + snapshot: () => snapshot(requests), + recordObservedSlotSize: vi.fn(), + subscribe: () => () => undefined, + }; + const bindings = { + get: () => ({ binding: { status: 'bound' as const }, element, visible: true }), + subscribe: () => () => undefined, + }; + + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + scheduleFrame: (callback) => callback(), + }); + + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 1, [728, 90]); + expect(requests[0].size).toEqual([1, 1]); + + getBoundingClientRect.mockReturnValue({ width: 970, height: 250 } as DOMRect); + ResizeObserverMock.instances.at(-1)!.emit(element); + expect(store.recordObservedSlotSize).toHaveBeenLastCalledWith(1, 1, [970, 250]); + observer.destroy(); + }); + + it.each(['unbound', 'ambiguous'] as const)('does not observe %s slots', (status) => { + const element = document.createElement('div'); + document.body.append(element); + const store = { + snapshot: () => snapshot([cycle(1, false)]), + recordObservedSlotSize: vi.fn(), + subscribe: () => () => undefined, + }; + const bindings = { + get: () => ({ binding: { status }, element, visible: false }), + subscribe: () => () => undefined, + }; + + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + scheduleFrame: (callback) => callback(), + }); + + expect(store.recordObservedSlotSize).not.toHaveBeenCalled(); + expect(ResizeObserverMock.instances.at(-1)!.observe).not.toHaveBeenCalled(); + observer.destroy(); + }); + + it('cannot apply a delayed prior-cycle measurement to a later refresh', () => { + const element = document.createElement('div'); + document.body.append(element); + vi.spyOn(element, 'getBoundingClientRect').mockReturnValue({ + width: 300, + height: 250, + } as DOMRect); + const requests = [cycle(1, false)]; + const listeners: Array<() => void> = []; + const store = { + snapshot: () => snapshot(requests), + recordObservedSlotSize: vi.fn(), + subscribe: (listener: () => void) => { + listeners.push(listener); + return () => undefined; + }, + }; + const bindings = { + get: () => ({ binding: { status: 'bound' as const }, element, visible: true }), + subscribe: () => () => undefined, + }; + const frames: Array<() => void> = []; + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + scheduleFrame: (callback) => frames.push(callback), + }); + const firstObserver = ResizeObserverMock.instances[0]; + + requests.push(cycle(2, false)); + listeners[0](); + frames.shift()!(); + firstObserver.emit(element); + while (frames.length > 0) frames.shift()!(); + + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 1, [300, 250]); + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 2, [300, 250]); + observer.destroy(); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts index 4c6721f3a..52aef6a7f 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts @@ -7,6 +7,7 @@ import { MAX_CALLBACK_ISSUES, MAX_CREATIVE_ATTEMPTS, MAX_DIAGNOSTIC_SLOTS, + MAX_REQUESTED_SLOT_SIZES, MAX_REQUEST_CYCLES_PER_SLOT, MAX_TRUSTED_SERVER_ASSOCIATIONS, REQUEST_PATH_ATTRIBUTION_WINDOW_MS, @@ -519,6 +520,38 @@ describe('GptDiagnosticsStore', () => { expect(cycle.responseClass).toBe('reservation'); }); + it('retains an observed outer slot box separately from GPT reported size', () => { + const store = new GptDiagnosticsStore({ now: () => 10 }); + const slot = fakeSlot('ad-slot-outer-box'); + + store.recordSlotRequested(slot); + store.recordSlotResponseReceived(slot); + store.recordSlotRenderEnded(slot, { isEmpty: false, size: [1, 1] }); + store.recordObservedSlotSize(1, 1, [728, 90]); + + const cycle = store.snapshot().slots[0].requests[0]; + expect(cycle.size).toEqual([1, 1]); + expect(cycle.observedSlotSize).toEqual([728, 90]); + }); + + it('rejects a stale prior-cycle outer-box measurement after a refresh', () => { + const store = new GptDiagnosticsStore({ now: () => 10 }); + const slot = fakeSlot('ad-slot-stale-outer-box'); + + store.recordSlotRequested(slot); + store.recordSlotResponseReceived(slot); + store.recordSlotRenderEnded(slot, { isEmpty: false }); + store.recordSlotRequested(slot); + store.recordSlotResponseReceived(slot); + store.recordSlotRenderEnded(slot, { isEmpty: false }); + store.recordObservedSlotSize(1, 1, [300, 250]); + store.recordObservedSlotSize(1, 2, [970, 250]); + + const requests = store.snapshot().slots[0].requests; + expect(requests[0].observedSlotSize).toBeUndefined(); + expect(requests[1].observedSlotSize).toEqual([970, 250]); + }); + it('separates a fill without Ad Manager identifiers from a reservation', () => { const store = new GptDiagnosticsStore({ now: () => 10 }); const slot = fakeSlot('ad-slot-default'); @@ -645,6 +678,62 @@ describe('GptDiagnosticsStore', () => { expect(cycles[1].trustedServerOpportunity).toBeUndefined(); }); + it('retains all configured requested slot sizes on only the correlated next request', () => { + const store = new GptDiagnosticsStore({ now: () => 10, defer: () => undefined }); + const slot = fakeSlot('requested-sizes'); + const formats: Array<[number, number]> = [ + [300, 250], + [728, 90], + [320, 50], + ]; + + store.recordTrustedServerOpportunity( + slot, + 'auction-slot', + 'renderable_candidate', + undefined, + formats + ); + formats[0]![0] = 1; + formats.push([970, 250]); + store.recordSlotRequested(slot); + store.recordSlotRequested(slot); + + const cycles = store.snapshot().slots[0]!.requests; + expect(cycles[0]?.requestedSlotSizes).toEqual([ + [300, 250], + [728, 90], + [320, 50], + ]); + expect(cycles[1]?.requestedSlotSizes).toBeUndefined(); + }); + + it('bounds and validates configured requested slot sizes before retaining them', () => { + const store = new GptDiagnosticsStore({ now: () => 10, defer: () => undefined }); + const slot = fakeSlot('validated-requested-sizes'); + const formats: Array<[number, number]> = Array.from( + { length: MAX_REQUESTED_SLOT_SIZES + 2 }, + (_, index) => [index + 1, 250] + ); + formats[0] = [0, 250]; + formats[1] = [300, Number.NaN]; + + store.recordTrustedServerOpportunity( + slot, + 'auction-slot', + 'renderable_candidate', + undefined, + formats + ); + store.recordSlotRequested(slot); + + const requested = store.snapshot().slots[0]!.requests[0]!.requestedSlotSizes; + expect(requested).toHaveLength(MAX_REQUESTED_SLOT_SIZES - 2); + expect(requested).not.toContainEqual([0, 250]); + expect(requested).not.toContainEqual([300, Number.NaN]); + expect(requested).not.toContainEqual([MAX_REQUESTED_SLOT_SIZES + 1, 250]); + }); + it('consumes a combined request intent with independent source facts', () => { let now = 10; const deferred: Array<() => void> = []; diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 9f9a3f977..8ead01aa8 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -414,9 +414,11 @@ describe('prebid/installPrebidNpm', () => { delete testWindow.__tsjs_prebid_diagnostics; delete testWindow.tsjs; delete mockPbjs['__tsApsBidResponseListenerInstalled']; + delete mockPbjs.bidderSettings; }); afterEach(() => { + delete mockPbjs.bidderSettings; vi.restoreAllMocks(); }); @@ -1130,6 +1132,35 @@ describe('prebid/installPrebidNpm', () => { }); describe('requestBids shim', () => { + it('preserves publisher ts adserverTargeting while adding trustedServer settings', () => { + const publisherTargeting = [{ key: 'ts', val: () => 'publisher-value' }]; + mockPbjs.bidderSettings = { + exampleBidder: { adserverTargeting: publisherTargeting }, + }; + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ bids: [{ bidder: 'exampleBidder', params: {} }] }], + } as unknown as RequestBidsArg); + + const bidderSettings = mockPbjs.bidderSettings as { + exampleBidder: { adserverTargeting: typeof publisherTargeting }; + trustedServer: { + allowAlternateBidderCodes: boolean; + allowedAlternateBidderCodes: string[]; + }; + }; + expect(bidderSettings.exampleBidder.adserverTargeting).toBe(publisherTargeting); + expect(bidderSettings.exampleBidder.adserverTargeting[0].key).toBe('ts'); + expect(bidderSettings.exampleBidder.adserverTargeting[0].val()).toBe('publisher-value'); + expect(bidderSettings.trustedServer).toEqual( + expect.objectContaining({ + allowAlternateBidderCodes: true, + allowedAlternateBidderCodes: ['*'], + }) + ); + }); + it('injects trustedServer bidder into every ad unit', () => { const pbjs = installPrebidNpm(); @@ -1850,25 +1881,33 @@ describe('prebid/installRefreshHandler', () => { it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { const originalRefresh = vi.fn(); - const clearTargeting = vi.fn(); + const slotTargeting = new Map([ + ['ts_initial', ['1']], + ['zone', ['homepage']], + ]); + const clearTargeting = vi.fn((key: string) => { + slotTargeting.delete(key); + }); + const setTargeting = vi.fn((key: string, value: string | string[]) => { + slotTargeting.set(key, Array.isArray(value) ? value : [value]); + }); const gptSlot = { getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), - getTargeting: vi.fn((key: string) => { - if (key === 'ts_initial') return ['1']; - if (key === 'zone') return ['homepage']; - return []; - }), + getTargeting: vi.fn((key: string) => slotTargeting.get(key) ?? []), getSizes: vi.fn(() => [ { getWidth: () => 970, getHeight: () => 250 }, { getWidth: () => 728, getHeight: () => 90 }, ]), clearTargeting, + setTargeting, }; const pubads = { refresh: originalRefresh, getSlots: vi.fn(() => [gptSlot]), }; - const setTargetingForGPTAsync = vi.fn(); + const setTargetingForGPTAsync = vi.fn(() => { + gptSlot.setTargeting('ts', 'prebid-value'); + }); mockPbjs.setTargetingForGPTAsync = setTargetingForGPTAsync; testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, @@ -1918,12 +1957,17 @@ describe('prebid/installRefreshHandler', () => { expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(clearTargeting).not.toHaveBeenCalledWith('ts'); expect(originalRefresh).not.toHaveBeenCalled(); const bidsBackHandler = mockRequestBids.mock.calls[0][0].bidsBackHandler; bidsBackHandler(); expect(setTargetingForGPTAsync).toHaveBeenCalled(); + expect(setTargetingForGPTAsync.mock.invocationCallOrder[0]).toBeLessThan( + originalRefresh.mock.invocationCallOrder[0] + ); + expect(slotTargeting.get('ts')).toEqual(['prebid-value']); expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); }); diff --git a/crates/trusted-server-js/src/bundle.rs b/crates/trusted-server-js/src/bundle.rs index 7ddc060ab..be5aa35cc 100644 --- a/crates/trusted-server-js/src/bundle.rs +++ b/crates/trusted-server-js/src/bundle.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::sync::OnceLock; +use std::sync::{Mutex, MutexGuard, OnceLock}; use hex::encode; use sha2::{Digest as _, Sha256}; @@ -10,7 +10,7 @@ include!(concat!(env!("OUT_DIR"), "/tsjs_modules.rs")); #[must_use] #[inline] pub fn module_bundle(id: &str) -> Option<&'static str> { - module_map().get(id).copied() + module_meta_map().get(id).map(|module| module.bundle) } /// Return all available module IDs, in discovery order (core first). @@ -27,56 +27,160 @@ pub fn all_module_ids() -> Vec<&'static str> { #[must_use] #[inline] pub fn concatenate_modules(ids: &[&str]) -> String { - let map = module_map(); - let mut parts: Vec<&str> = Vec::new(); + let ordered_ids = concatenated_module_ids(ids); + let mut body = String::new(); + visit_concatenated_module_parts(&ordered_ids, |part| body.push_str(part)); + body +} + +/// SHA-256 hash of the concatenated modules, for cache-busting URLs. +/// +/// The hash is computed over the same byte sequence as [`concatenate_modules`] +/// without allocating that concatenated body. Results are memoized by ordered +/// module ID list for reused processes or isolates. Fastly creates a fresh Wasm +/// instance per request, but still benefits from hashing without materializing +/// the concatenated body. +#[must_use] +#[inline] +pub fn concatenated_hash(ids: &[&str]) -> String { + let key = concatenated_module_ids(ids); + if let Some(hash) = lock_concatenated_hash_cache().get(&key).cloned() { + return hash; + } + + let hash = hash_concatenated_modules(&key); + lock_concatenated_hash_cache().insert(key, hash.clone()); + hash +} + +/// SHA-256 hash of a single module's content (without prepending core). +/// +/// Used for cache-busting URLs of deferred modules served individually. +#[must_use] +#[inline] +pub fn single_module_hash(id: &str) -> Option<&'static str> { + module_meta_map().get(id).map(|module| module.sha256) +} + +fn concatenated_module_ids(ids: &[&str]) -> Vec<&'static str> { + let map = module_meta_map(); + let mut ordered = Vec::new(); - // Core always first if let Some(core) = map.get("core") { - parts.push(core); + ordered.push(core.id); } - // Then requested modules (excluding core, already included) for id in ids { if *id == "core" { continue; } - if let Some(bundle) = map.get(id) { - parts.push(bundle); + if let Some(module) = map.get(*id) { + ordered.push(module.id); } } - parts.join(";\n") + ordered } -/// SHA-256 hash of the concatenated modules, for cache-busting URLs. -#[must_use] -#[inline] -pub fn concatenated_hash(ids: &[&str]) -> String { - let body = concatenate_modules(ids); +fn hash_concatenated_modules(ids: &[&'static str]) -> String { let mut hasher = Sha256::new(); - hasher.update(body.as_bytes()); + visit_concatenated_module_parts(ids, |part| hasher.update(part.as_bytes())); encode(hasher.finalize()) } -/// SHA-256 hash of a single module's content (without prepending core). -/// -/// Used for cache-busting URLs of deferred modules served individually. -#[must_use] -#[inline] -pub fn single_module_hash(id: &str) -> Option { - module_bundle(id).map(|content| { - let mut hasher = Sha256::new(); - hasher.update(content.as_bytes()); - encode(hasher.finalize()) - }) +fn visit_concatenated_module_parts(ids: &[&'static str], mut visit: F) +where + F: FnMut(&'static str), +{ + let map = module_meta_map(); + let mut first = true; + + for id in ids { + let Some(module) = map.get(*id) else { + continue; + }; + if first { + first = false; + } else { + visit(";\n"); + } + visit(module.bundle); + } } -fn module_map() -> &'static HashMap<&'static str, &'static str> { - static MAP: OnceLock> = OnceLock::new(); +fn module_meta_map() -> &'static HashMap<&'static str, &'static TsjsModuleMeta> { + static MAP: OnceLock> = OnceLock::new(); MAP.get_or_init(|| { TSJS_MODULES .iter() - .map(|module| (module.id, module.bundle)) + .map(|module| (module.id, module)) .collect() }) } + +fn lock_concatenated_hash_cache() -> MutexGuard<'static, HashMap, String>> { + match concatenated_hash_cache().lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +fn concatenated_hash_cache() -> &'static Mutex, String>> { + static CACHE: OnceLock, String>>> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sha256_hex(bytes: &[u8]) -> String { + encode(Sha256::digest(bytes)) + } + + #[test] + fn generated_single_module_hashes_match_bundle_contents() { + for id in all_module_ids() { + let bundle = module_bundle(id).expect("should have module bundle"); + let generated_hash = single_module_hash(id).expect("should have generated hash"); + + assert_eq!( + generated_hash, + sha256_hex(bundle.as_bytes()), + "generated hash for module {id} should match included bundle bytes" + ); + } + } + + #[test] + fn concatenated_hash_matches_concatenated_bundle_contents() { + let available_ids = all_module_ids(); + let non_core_ids = available_ids + .iter() + .copied() + .filter(|id| *id != "core") + .take(3) + .collect::>(); + + let mut cases: Vec> = vec![Vec::new()]; + if let Some(first) = non_core_ids.first().copied() { + cases.push(vec![first]); + } + if non_core_ids.len() >= 2 { + cases.push(non_core_ids[..2].to_vec()); + cases.push(non_core_ids[..2].iter().rev().copied().collect()); + } + if non_core_ids.len() >= 3 { + cases.push(non_core_ids[..3].to_vec()); + } + + for ids in cases { + let concatenated = concatenate_modules(&ids); + assert_eq!( + concatenated_hash(&ids), + sha256_hex(concatenated.as_bytes()), + "concatenated hash should match concatenated bundle bytes for {ids:?}" + ); + } + } +} diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index b4ba6b797..6fa5cc9f3 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -823,3 +823,72 @@ Every auction response includes structured metadata in `ext.orchestrator`: "time_ms": 145 } ``` + +### SSAT HTML Debug Comment + +For local server-side auction template (SSAT) investigation, Trusted Server can +insert a `` comment before the page's bids script. Enable +it in `trusted-server.toml`, push the local configuration, restart the local +server, and search the page source for `ts-debug`: + +```toml +[debug] +auction_html_comment = true + +[debug.auction_html_comment_options] +include_provider_responses = true +include_mediator_response = false +include_bids = false +metadata_keys = ["error_type", "http_status", "message"] +verbosity = "full" +format = "pretty" +``` + +```bash +ts config validate +ts config push --adapter fastly --local +fastly compute serve +``` + +This example is useful when investigating raw Prebid Server requests and +responses without spending the dump budget on winning creatives. Raw PBS +`debug.httpcalls` and `resolvedrequest` metadata also require +`debug = true` under `[integrations.prebid]`. + +| Option | Default | Behavior | +| ---------------------------- | -------------------------------------- | ----------------------------------------------------------------------- | +| `include_provider_responses` | `true` | Include the provider response array | +| `include_mediator_response` | `true` | Include the mediator response when a mediator ran | +| `include_bids` | `true` | Include bid objects; when `false`, provider status and metadata remain | +| `metadata_keys` | `error_type`, `http_status`, `message` | Select a subset of the fixed validated metadata keys in `redacted` mode | +| `verbosity` | `redacted` | Select `redacted`, `upstream`, or `full` sensitivity | +| `format` | `compact` | Use compact outer JSON or indented outer JSON with `pretty` | + +The verbosity modes form an explicit sensitivity ladder: + +- `redacted` reconstructs only validated `error_type`, `http_status`, and a + server-generated `message`, intersected with `metadata_keys`. A successful + provider response can therefore have `metadata: {}`. +- `upstream` adds provider-controlled errors, warnings, response timings, bid + statuses, and bounded upstream-message fields. It does not include raw PBS + `httpcalls` or `resolvedrequest`. +- `full` includes raw response metadata and untruncated creatives. It can expose + IP addresses, geo data, identifiers, consent strings, request signatures, and + complete provider request/response bodies. + +`format = "pretty"` indents only the outer dump. JSON-looking fields such as +`requestbody` and `responsebody` remain strings exactly as captured, so their +contents still appear escaped. Use a local JSON inspection tool when those +nested values need additional formatting. + +The summary line's `winning=N` count is computed before section filtering, so +it can be nonzero while `include_bids = false` produces empty bid arrays. Every +mode and format neutralizes HTML-comment terminators and enforces a 256 KiB +total dump cap. A capped dump ends with `…(truncated N bytes)` and is no longer +valid JSON. + +::: danger Local debugging only +Do not enable the auction HTML comment in production. Even `redacted` can +contain bid-level data and creative previews, while `upstream` and `full` may +expose identity-bearing request data to anyone who can view the page source. +::: diff --git a/docs/guide/cli.md b/docs/guide/cli.md index b6829895e..9c31fb56d 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -98,7 +98,7 @@ Chrome or Chromium must be installed locally. The command checks common PATH names and standard macOS/Linux install locations. ```bash -ts audit https://publisher.example +ts audit generate https://publisher.example ``` By default, the command writes: @@ -118,13 +118,13 @@ ts config validate If a config already exists, avoid overwriting it: ```bash -ts audit https://publisher.example --no-config +ts audit generate https://publisher.example --no-config ``` Use custom output paths when reviewing artifacts first: ```bash -ts audit https://publisher.example \ +ts audit generate https://publisher.example \ --js-assets audit/js-assets.toml \ --config audit/trusted-server.toml ``` @@ -132,7 +132,232 @@ ts audit https://publisher.example \ Use `--force` only when replacing existing output files is intentional: ```bash -ts audit https://publisher.example --force +ts audit generate https://publisher.example --force +``` + +The legacy `ts audit ` form remains a compatibility alias for artifact +generation. New automation should use `ts audit generate `. + +## Generate ad-template slots from a live site + +`ts audit ad-templates generate ` discovers the publisher's ad slots and +rewrites the `[creative_opportunities]` slot array in `trusted-server.toml` in +place, preserving every other section and comment. + +```bash +ts audit ad-templates generate https://publisher.example/ +``` + +It samples the site rather than a single page. Ad slots repeat per site +section, so the crawl is sized by the publisher's taxonomy — a dozen sections — +not its catalogue: + +1. Load the requested page and read its links and, from `robots.txt`, its + sitemap. +2. Group both into candidate sections, keeping one landing page and one article + per section. +3. Load those pages, recording each slot's div, sizes, and GAM ad-unit path. +4. Reconcile every slot across the pages it appeared on. +5. Infer a `{section}` ad-unit template if the evidence proves one. +6. Verify the result loads, then write it. + +### What it writes + +Given a site whose ad units track the section, the run produces: + +```toml +[creative_opportunities] +gam_network_id = "99999" +section_root = "homepage" +section_segment = 0 + +[[creative_opportunities.slot]] +id = "ad-header-0" +div_id = "ad-header-0" +gam_unit_path = "/{network_id}/example/{section}" +page_patterns = ["/", "/deals", "/deals/*", "/news", "/news/*"] +formats = [{ width = 728, height = 90 }] +``` + +Each section contributes **two** patterns. `*` crosses `/` in this glob +dialect, so `/news/*` matches `/news/a/b` but not the bare `/news` landing +page; emitting only the star form would drop the landing page from the slot. + +Sizes are unioned across pages, so a format that renders only on articles +survives alongside the homepage's. + +### When it keeps literal paths, and when it refuses + +A wrong ad-unit template makes the publisher bid against inventory that does not +exist, so the command prefers a narrow literal path over a plausible guess. + +| Situation | Result | +| --------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| Only one page was crawled | Literal path. One observation cannot distinguish a literal from a template. | +| The ad unit never varied by section | Literal path. | +| A section's slug is not derivable from its URL (`/car-research` requesting `.../carresearch`) | Literal path; the round-trip check catches it. | +| No root page was seen, so `section_root` is unknown | Literal path rather than a guessed fallback. | +| Two path segments could both be the section | No template; the ambiguity is reported. | +| The ad unit varies by device, geo, or anything the URL cannot supply | **No `gam_unit_path` at all** for that slot. | +| Crawled pages report different GAM network ids | The run fails; the pages are not one property. | +| More than a quarter of crawled pages return no slots | The run fails. That is the signature of bot protection serving challenge pages, and writing from it would silently narrow the slot set. | + +Every run checks that the config it produced still loads before replacing the +file, and `--dry-run` runs the same check — a clean preview is evidence the +config loads, not just that it parses. + +### Bounding and steering the crawl + +```bash +# Cover more of a large site. +ts audit ad-templates generate https://publisher.example/ --max-sections 20 --max-pages 41 + +# Audit exactly one page, as earlier releases did. +ts audit ad-templates generate https://publisher.example/ --max-pages 1 + +# Set the patterns yourself; this disables pattern inference entirely. +ts audit ad-templates generate https://publisher.example/ \ + --page-pattern '/' --page-pattern '/news' --page-pattern '/news/*' + +# Preview without writing. +ts audit ad-templates generate https://publisher.example/ --dry-run +``` + +Re-running merges into the existing slots: a slot seen again keeps its +hand-tuned fields and gains this run's patterns, and a hand-written +`gam_unit_path` template is preserved. `--replace` discards existing slots +instead, which also discards any template you wrote by hand. + +Behind bot protection, pass a valid clearance cookie. The crawl reuses one +browser session, so clearance earned on the first page carries to the rest, and +`--page-delay-ms` spaces the requests — an unpaced crawl is both discourteous to +the origin and likelier to be challenged partway through: + +```bash +ts audit ad-templates generate https://publisher.example/ \ + --cookie '=' --page-delay-ms 1500 +``` + +Some origins refuse a headless browser outright regardless of the cookie. +`--headful` runs a visible one, which is also the quickest way to _see_ whether +a challenge is being shown: + +```bash +ts audit ad-templates generate https://publisher.example/ --headful +``` + +### Sites behind a consent platform + +Publishers gate slot definition behind their consent platform, and the audit +runs in a throwaway browser profile with no consent cookie. Left alone, such a +site defines no slots at all and looks identical to a site with no ad stack. + +The crawl therefore answers the two IAB interfaces every compliant platform +exposes — TCF v2 and US Privacy — as a consenting, out-of-scope reader, before +any page script runs. This changes only what the audit browser sees; it does not +affect the publisher's own readers. Pass `--no-assume-consent` to observe the +un-consented page instead. + +When a page still yields no slots, the run reports GPT's observable state — +whether the library reached `apiReady`, how many queued commands never drained, +how many scripts ran. An empty slot registry has several very different causes, +and that line distinguishes them. + +### Auditing a production hostname served locally + +`ts dev proxy` serves a production hostname from a local Trusted Server. +Auditing through it keeps the page's origin, cookie scope, and any origin checks +in the ad stack matching production rather than `localhost`: + +```bash +ts dev proxy --map www.publisher.example=127.0.0.1:7676 --upstream-plaintext --rewrite-host + +ts audit ad-templates generate https://www.publisher.example/ \ + --browser-proxy 127.0.0.1:18080 --danger-accept-invalid-certs +``` + +`--danger-accept-invalid-certs` covers the proxy's MITM certificate when the +throwaway browser profile does not trust its CA; installing that CA +(`ts dev proxy ca`) is preferable. Against a real origin the flag is dangerous — +the audit sends any `--cookie` session upstream and treats the response as +evidence, so an invalid certificate could mean an impersonator is both +harvesting the session and fabricating the result. + +Note that a local Trusted Server injects its own configured slots into the page, +so a run through the proxy can rediscover config it already has. Slot ids that +are absent from the current config are the publisher's own. + +### Slots that change div id on every render + +Some ad stacks build div ids from a per-render token, so one placement arrives +under a new id on every page. Those ids match nothing at runtime, so the run +declines to write them and reports the group instead: + +```text +note: skipped 3 slot(s) that look like one placement under a per-render div id + on `/12345678/example.com_Overlay` (kso_2632930aBc_overlay_1, …); + they share the prefix `kso`. Add it once by hand with a div_id prefix + that is stable across renders +``` + +The detection is by evidence, not by recognising token shapes: candidates share +an ad-unit path and formats, and what separates a fragmented placement from two +legitimate siblings on one unit is co-occurrence — real siblings appear together +on a page, fragments never do. The suggested prefix is a starting point only, not +written as a `div_id`, because it reaches only as far as the observed tokens +happen to agree. + +### Checking for a device split + +Publishers often serve a different ad unit per device +(`/network/desktop/news` against `/network/mobile/news`). A desktop-only crawl +cannot see that — it infers a template correct for desktop and silently wrong +for every mobile impression. + +```bash +ts audit ad-templates generate https://publisher.example/ --profiles desktop,mobile +``` + +Each page is loaded once per profile. Where the profiles disagree, the slot is +written with its div and formats but **no** `gam_unit_path`, so the runtime +falls back to the default unit rather than bidding on one that does not exist. + +### Deploy ordering for templated config + +> **A config containing `section_root` or `section_segment` is not +> rollback-safe.** These keys are rejected outright by a Trusted Server binary +> that predates ad-unit templating, and the rejection fails the _entire_ +> configuration load — not just the ad-template section — so every route serves +> an error. This is a full-site outage, not a degraded ad stack. + +When a run reports that it wrote a `{section}` template: + +1. Deploy the template-aware binary **first**. +2. Then `ts config push`. +3. Do **not** roll that binary back while the config is live. + +A run that did not template writes neither key, and leaves the config exactly as +rollback-safe as it was. + +### Audit safety defaults + +Every `ts audit` browser session validates TLS certificates. This matters +because `--cookie` sends a real session to the origin and the page's own +response becomes the audit's evidence, so a certificate-invalid host could both +harvest the session and fabricate what the audit reports. Override only for a +host you control with a known self-signed certificate: + +```bash +ts audit page https://staging.publisher.example --danger-accept-invalid-certs +``` + +`ts audit ad-templates verify` matches configured slots against the +**post-redirect** path, so it refuses a redirect that leaves the requested +origin rather than accepting another site's evidence as verification. Allow it +for a known redirect between your own properties (for example apex to `www`): + +```bash +ts audit ad-templates verify https://publisher.example/ --allow-cross-origin-redirect ``` `ts audit` is not an EdgeZero adapter command. It has no `--adapter` option and diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index ddb6544ce..28aae2afe 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -72,6 +72,7 @@ fail and the service will return its startup-error response. | `[ec]` | Edge Cookie (EC) ID generation | | `[tester_cookie]` | Optional tester-cookie endpoint | | `[proxy]` | Proxy SSRF allowlist and asset routes | +| `[cache]` | Static/rehosted asset cache policy rules | | `[image_optimizer]` | Reusable Image Optimizer profile sets | | `[request_signing]` | Ed25519 request signing | | `[auction]` | Auction orchestration | @@ -1031,6 +1032,129 @@ when_missing = "smart" See [Asset Routes](/guide/asset-routes) for request flow, S3 auth details, and Image Optimizer behavior. +## Cache Configuration + +Static and rehosted asset cache upgrades are operator-controlled. By default, +Trusted Server leaves arbitrary publisher-origin assets under origin cache +control. Add `[[cache.asset_rules]]` entries only for paths that are known to be +content-addressed or otherwise safe for the configured TTL. + +### `[[cache.asset_rules]]` + +Rules are evaluated in file order; the first enabled matching rule wins. +Disabled rules never match, and their matcher and policy validation is deferred +until they are enabled. Rule IDs are always normalized and must remain nonempty +and unique, including for disabled placeholders. + +| Field | Type | Required | Description | +| -------------------------------- | ------------- | -------- | ---------------------------------------------------------------------------------- | +| `id` | String | Yes | Unique operator-facing rule identifier | +| `enabled` | Boolean | No | Whether the rule participates in matching (default `false`) | +| `preset` | String | Matcher | Built-in preset such as `nextjs-static` | +| `path_prefix` | String | Matcher | Request path prefix | +| `path_glob` | String | Matcher | Single glob matched against the request path | +| `path_globs` | Array[String] | Matcher | Multiple globs matched against the request path | +| `path_regex` | String | Matcher | Regex matched against the request path | +| `extensions` | Array[String] | Matcher | Case-insensitive file extensions | +| `fingerprint_style` | String | No | Required bundler fingerprint convention before matching | +| `visibility` | String | No | `public` or `private` (default `public`) | +| `browser_ttl_seconds` | Integer | Policy | Browser `max-age`; required for private rules and positive with `immutable = true` | +| `edge_ttl_seconds` | Integer | Policy | Public rules only: TTL emitted through the runtime-specific shared-cache directive | +| `stale_while_revalidate_seconds` | Integer | No | Optional `stale-while-revalidate` | +| `stale_if_error_seconds` | Integer | No | Optional `stale-if-error` | +| `immutable` | Boolean | No | Add `immutable` for a validated content-addressed rule | + +An enabled rule must configure exactly one matcher. Public rules must configure +at least one of `browser_ttl_seconds` or `edge_ttl_seconds`; private rules must +configure `browser_ttl_seconds` and must not configure `edge_ttl_seconds`. +`path_glob` and `path_globs` are mutually exclusive. `immutable = true` +additionally requires a positive browser TTL and either the content-addressed +`nextjs-static` preset or an explicit `fingerprint_style`. + +The filename fingerprint check is intentionally conservative and style-specific. +It examines the suffix immediately before the final extension and requires a +nonempty filename prefix separated by `.`, `-`, `_`, or `~`. Set exactly the +style emitted by the publisher's bundler: + +- `hex`: hexadecimal suffixes of at least eight characters containing a letter, + such as `app.0123abcd.js`; +- `esbuild-base32`: eight-character uppercase Base32 suffixes, such as + `app-VRTVD5R5.js`; +- `vite-base64-url`: eight-character Base64URL suffixes with a mixed character + class, such as `index-BsELY24f.js`. + +A style is an explicit operator assertion, not proof of content addressing. +For example, some human-written mixed-case names can resemble a Vite suffix, so +only select `vite-base64-url` after verifying the publisher's build output. A +base rule that matches while its selected fingerprint style fails emits a debug +log with the rule ID and rejected path. + +Glob patterns are case-sensitive. `*` matches within a single path component, +while `**` matches recursively: `/assets/*.js` matches `/assets/app.js` but not +`/assets/vendor/app.js`; `/assets/**/*.js` matches both. + +**Next.js preset example** (disabled until the publisher confirms +`/_next/static/` is content-addressed): + +```toml +[[cache.asset_rules]] +id = "nextjs-static" +enabled = false +preset = "nextjs-static" +visibility = "public" +browser_ttl_seconds = 31536000 +edge_ttl_seconds = 31536000 +immutable = true +``` + +**Publisher allowlist example** (enable only after verifying the filename +convention): + +```toml +[[cache.asset_rules]] +id = "publisher-fingerprinted-assets" +enabled = false +path_globs = [ + "/assets/**/*.js", + "/assets/**/*.css", + "/assets/**/*.png", + "/assets/**/*.webp", +] +fingerprint_style = "vite-base64-url" +visibility = "public" +browser_ttl_seconds = 31536000 +edge_ttl_seconds = 31536000 +immutable = true +``` + +If `[cache]` is omitted or no enabled rule matches, Trusted Server preserves the +origin cache policy for publisher-origin assets. On the publisher pass-through +path, an origin `private` or `no-store` directive vetoes a matching rule. Other +origin cache directives, including `no-cache`, are replaced by the configured +policy. `Vary` is preserved, so do not assign a public immutable rule to paths +that vary by cookies or other user-specific request state. + +On a configured Fastly asset-rehost route, a matching rule is authoritative +over the third-party origin's cache defaults, including `no-store`, because +Trusted Server owns the rehosted copy. A later Trusted Server or operator-applied +`private` or `no-store` directive still vetoes public policy reapplication and +removes shared-cache headers. + +TS-owned validated hash URLs such as `/static/tsjs=...js?v=` use their +built-in cache policy and do not require an asset rule. Shared-cache keys for +`/static/tsjs=` must preserve `v`; otherwise a matching immutable response can +collide with the missing or mismatched version's short-TTL response. + +`edge_ttl_seconds` only emits the selected runtime's shared-cache directive for +public rules. The runtime or service must also enable and consume that +directive. The checked-in Cloudflare manifests intentionally do not enable +Workers Cache: the Worker serves the full publisher gateway, not an isolated +static-only entrypoint. Emitting `Cloudflare-CDN-Cache-Control` alone must not +be treated as permission to cache every response. Any future Workers Cache +opt-in must isolate or explicitly allowlist cacheable traffic. Fastly synthetic +and final egress responses still require explicit runtime cache integration, +tracked in [#908](https://github.com/IABTechLab/trusted-server/issues/908). + ## Integration Configurations Settings for built-in integrations (Prebid, Next.js, Osano, Permutive, Testlight). For other @@ -1347,8 +1471,16 @@ Defines the ad slots the trusted server offers on a page: which pages each slot appears on (`page_patterns`), its supported sizes (`formats`), and the GAM ad unit it maps to (`gam_unit_path`). +`enabled` is the dedicated server-side ad-template switch. It defaults to `true` +for compatibility with existing configurations. Set it to `false` to stop +publisher HTML and SPA page-bids template delivery while retaining the slot +configuration and direct `POST /auction` endpoint. The browser-facing cache +policy for a disabled template stack is `Cache-Control: max-age=60`, unless the +origin already sends `private` or `no-store`. + ```toml [creative_opportunities] +enabled = true # set to false to disable server-side ad templates gam_network_id = "123456789" price_granularity = "dense" @@ -1367,6 +1499,129 @@ page_patterns = ["/", "/news", "/news/*", "/reviews", "/reviews/*"] formats = [{ width = 728, height = 90 }] ``` +The same switch can be overridden through the legacy environment-variable +loader: + +```bash +TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false +``` + +### Shared template assembly (`assembly_mode = "esi"`) + +`assembly_mode` controls how initial-page slot and bid state is delivered: + +- `inline` (default) transforms every origin response and injects the current + reader's slots and bids directly. +- `esi` opts into a reader-neutral transformed-template cache on Fastly. The + cache stores identity bytes containing one inert, versioned comment. On an + authorized cold miss, Fastly replaces that comment in a private working copy + with one synthetic ESI include and resolves it from the already-built reader + state using the pinned `stackpop/esi` parser. No HTTP fragment request occurs. + Warm hits use an exact byte split instead, preserving the fast article-prefix + stream while the auction finishes. + +This is deliberately not general publisher-controlled ESI. A transformed origin +document containing any `` head marker selecting the publisher hook below. The marker works under a publisher CSP that blocks inline scripts. Unknown values fail configuration deserialization. + +### Publisher-native hook experiment + +`publisher_native` is an opt-in publisher integration seam, **not** APS compatibility proof. No public APS API was found that accepts an externally selected OpenRTB `aaxResponse` for native rendering. In particular, `apstag.setDisplayBids()` operates on APS's own `fetchBids()` state and is not an ingestion API for the exact Trusted Server-selected bid. Trusted Server does not call `apstag`, `fetchBids`, or `setDisplayBids`, mutate APS internals, or start a second auction. + +Before Trusted Server JS receives a selected descriptor, the publisher must install this versioned hook: + +```js +window.tsjs = window.tsjs || {} +window.tsjs.apsNativeRenderer = { + render({ version, slotId, renderer }) { + // version is exactly 1; renderer is frozen and fully validated. + // Render only this exact selected descriptor through publisher-owned logic. + return { accepted: true } + }, +} +``` + +The hook receives exactly `{ version: 1, slotId, renderer }`. It must return or resolve an object with `accepted: boolean`; `reason?: string` is allowed for fictional-safe diagnostics. Missing hooks, throws, rejected promises, malformed acknowledgements, `{ accepted: false }`, and acknowledgements that take longer than 10 seconds visibly decline the bid. They never fall back to the Trusted Server iframe or send a Universal Creative renderer response. A newer dispatch makes an older acknowledgement stale and ignored; late acknowledgements after the timeout are also ignored. Hook implementations own cancellation of any already-started side effect and actual render completion. + +For a client-side Prebid APS capability, Trusted Server consumes the one-shot capability before delegation and calls `markWinningBidAsUsed` only after `accepted: true`. For server/GPT ownership, it similarly claims the slot/ad ID before invoking the hook. This prevents native and Trusted Server rendering from both owning the same response, but an accepting hook is responsible for real rendering semantics. + +Disable or coordinate publisher-native APS demand for every `publisher_native` cohort. Otherwise native APS demand and this server-selected bid can duplicate demand. Validate a controlled real publisher/account setup with the APS account team before any production rollout. + Set `inventory_domain` and `inventory_page_origin` together only when the public deployment hostname differs from the inventory identity authorized by APS. The domain becomes `site.domain`. The HTTPS page origin replaces the current page's scheme and host while preserving its path; query and fragment data are removed before forwarding. The origin must be the inventory domain or one of its subdomains and cannot include credentials, a port, path, query, or fragment. These values come only from operator configuration; Trusted Server never accepts APS inventory identity from the client auction payload. APS uses ordinary auction slot IDs and banner formats. Legacy creative-opportunity APS `slot_id` configuration is accepted for compatibility but ignored, and `bidders.aps.slotID` is not required. Remove both during migration. @@ -150,7 +177,7 @@ Trusted Server does not insert APS creative markup into the publisher document. Seats, `impid`, markup, notifications, user-sync data, sibling bids, losing seats, and unknown fields are not exposed. The browser decodes this envelope and cross-checks the ID, dimensions, URL, and tag type before any DOM mutation or message suppression. -Both rendering paths use `GET /integrations/aps/renderer`, a static Trusted Server document with its own restrictive CSP. The document initializes the account-keyed APS queue and then loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. +In `trusted_server` mode, both rendering paths use `GET /integrations/aps/renderer`, a static Trusted Server document with its own restrictive CSP. The document initializes the account-keyed APS queue and then loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. The outer iframe uses these sandbox permissions: @@ -167,11 +194,11 @@ It deliberately omits `allow-same-origin`, so APS and bidder execution remains b ### Direct `/auction` -The TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. +In `trusted_server` mode, the TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. In `publisher_native` mode it instead calls the explicit publisher hook and creates no Trusted Server iframe. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. ### GAM and Universal Creative -For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program that creates the same opaque renderer iframe. +For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. In `publisher_native` mode the ownership-checked bridge calls the publisher hook without sending a Universal Creative renderer response; in `trusted_server` mode it uses the static dynamic renderer described below. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program that creates the same opaque renderer iframe. For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes the capability once, and passes the APS bid ID separately to the Amazon runner. diff --git a/docs/guide/integrations/gpt-diagnostics.md b/docs/guide/integrations/gpt-diagnostics.md index 78c657bf6..1f4efbf22 100644 --- a/docs/guide/integrations/gpt-diagnostics.md +++ b/docs/guide/integrations/gpt-diagnostics.md @@ -19,7 +19,7 @@ Server candidate, a PUC markup request, a successfully posted markup response, a GPT slot load are separate steps in an evidence ladder. This feature requires zero publisher-code changes. Activation remains the existing -server integration configuration plus `?ts_console=true`; it does not require new +server integration configuration plus `?ts_console=1`; it does not require new publisher JavaScript, React, Next.js, DOM, or GAM configuration. The diagnostics integration is independent of the @@ -36,11 +36,36 @@ The module is unavailable unless explicitly enabled for the deployment: enabled = true ``` -Deployment configuration only makes the module available. Inactive browser sessions -receive no diagnostics module. When activated, the standalone content-hashed module -loads synchronously after the core bundle so it can install listeners before -publisher GPT request code. The standalone static response is cookie-independent and -remains publicly cacheable; active HTML responses are private and non-storeable. +Deployment configuration makes the module available; it does not activate any browser +session. Inactive browser sessions receive no diagnostics module. When activated, the +standalone content-hashed module loads synchronously after the core bundle so it can +install listeners before publisher GPT request code. The standalone static response is +cookie-independent and remains publicly cacheable; active HTML responses are private +and non-storeable. + +### Auction correlation token + +Enabling the integration has one server-side effect that does not depend on browser +activation. For each server-side auction that produced winning bids, Trusted Server +mints a fresh correlation token and publishes it as `hb_auction_id` on each winning bid +in `window.tsjs.bids`: + +```text +ts-auc-2f8c1d5a4b7e4c0f9a3d6b1e8c5f2a7d +``` + +- The token is generated per auction from a random UUID. It is not derived from the Edge + Cookie ID, the auction request ID, or any other visitor identifier, and it does not + repeat across auctions. +- It is **not** a GAM targeting key. Of the Trusted Server bid fields, only `hb_pb`, + `hb_bidder`, `hb_adid`, `hb_cache_host`, `hb_cache_path`, and `ts_initial` are applied + as slot targeting alongside the slot's configured targeting, so the token never enters + the ad request. +- It is absent when the integration is disabled, and absent for any auction that + produced no winning bids. +- It is published on every document whose auction produced winning bids, including + documents with no active console session, because the console reads it from the same + page bid state the GPT integration already consumes. ## Activate or Deactivate a Browser Session @@ -56,7 +81,13 @@ Open a page with one of these exact, case-sensitive query directives: For example: ```text -https://publisher.example.com/article?ts_console=true +https://publisher.example.com/article?ts_console=1 +``` + +Deactivate the same browser session with the matching directive: + +```text +https://publisher.example.com/article?ts_console=0 ``` An exact directive establishes or clears the host-only, `Secure`, `HttpOnly`, @@ -89,7 +120,7 @@ Each request cycle can show: - GPT slot-onload, impression-viewable, and visibility observations. - Non-negative request-to-response, response-to-render, render-to-load, and render-to-viewable durations. -- Rendered size, backfill, and slot-content-change facts exposed by GPT. +- GPT-reported rendered size, a separately labelled observed outer slot box when safely bound, backfill, and slot-content-change facts. - Current DOM binding status and viewport intersection. Elapsed time alone never changes a pending GPT request to Incomplete. Incomplete @@ -124,10 +155,11 @@ arguments, result, and synchronous throw. A `refresh()` call that omits its slot list, or passes `null` or `undefined` for it, refreshes every slot; the observer reads GPT's current slot list for diagnostics only. A stale refresh function reference captured before installation bypasses that boundary and remains -`unattributed`. Prebid sets a scoped, -synchronous diagnostics context while delegating its own refresh, so nesting does not -mislabel a Prebid refresh as `competing`. Diagnostics never suppresses or changes a -GPT request. +`unattributed`. Prebid and the Trusted Server `adInit` refresh each set a scoped, +synchronous diagnostics context while delegating their own refresh, so a nested +`pubads.refresh` is not mislabeled `publisher_refresh` or `competing`. Both clear that +context even when the delegated refresh throws, and the Prebid wrapper restores the +exact prior value. Diagnostics never suppresses or changes a GPT request. For a direct observation, the optional opaque auction ID is retained only after trimming to a non-empty value no longer than 256 UTF-8 bytes. No auction payload, @@ -191,10 +223,12 @@ the state. A matched creative attempt can report these safe, non-terminal categories: -- `missing_render_source` -- `cache_fetch_failed` -- `invalid_cache_payload` -- `response_post_failed` +| Failure | Observed at the bridge | +| ----------------------- | ---------------------------------------------------------------------------------------------- | +| `missing_render_source` | The bid carried neither inline markup nor a complete PBS Cache host and path. | +| `cache_fetch_failed` | The PBS Cache fetch was rejected or returned a non-OK status. | +| `invalid_cache_payload` | The cache response was read but held no renderable creative, so nothing was posted. | +| `response_post_failed` | `port.postMessage` threw while posting markup, on either the inline or the cached-markup path. | Failures are deduplicated and retain first-observed order. Detailed URLs, cache IDs, payloads, markup, and error objects remain only in existing operational logging and do @@ -219,6 +253,11 @@ line-item backfill alike, so they classify as `reservation` only when GPT also reported the render as explicitly non-backfill. On their own they remain `unclassified_non_empty` rather than becoming an unsupported conclusion. +Identifiers are retained only as positive whole numbers, and the yield-group and company +lists keep at most eight IDs each. GPT reports these fields only for reservation and +backfill ads served by PubAdsService, so an absent identifier is a fact about the render +rather than a gap in observation. + ## Attribution Issues and Callback Coverage Creative-correlation problems are exported separately from GPT callback issues. The @@ -270,12 +309,58 @@ A concise viewport badge appears only when a slot: - Has a unique, connected exact binding. - Has a non-zero rectangle intersecting the viewport. +A badge summarizes the slot's most recent request cycle: the GPT result (Filled, Empty, +Rendered (fill unknown), or Pending), a short delivery label, a `Competing paths` +marker when the request path is `competing`, the rendered size, and the request-to- +response, response-to-render, and render-to-viewable durations that are available. It +adds `Incomplete sequence` when a callback proved a missing or invalid earlier step. + +Badge delivery labels are the same derived states the panel and export report, shortened +to fit: + +| Delivery state | Badge label | +| ------------------------------ | ----------------------- | +| `trusted_server_response_sent` | TS response sent | +| `trusted_server_selected` | TS selected | +| `pending` | TS candidate (pending) | +| `candidate_unconfirmed` | TS unconfirmed | +| `no_candidate` | No TS candidate | +| `unknown` | Delivery unknown | +| `not_applicable` | No delivery label shown | + +The badge never re-derives delivery from raw timestamps; it labels the state the store +already resolved, so a badge cannot disagree with the panel or the export. + Missing elements and duplicate DOM or GPT slot IDs remain visible in the panel as Unbound or Ambiguous and receive no badge. If DOM uniqueness cannot be verified because selector support is unavailable or throws, the export reports `dom_uniqueness_unverifiable`. Framework replacement of an element with a new unique element using the same exact ID is rebound automatically. +When Trusted Server associates a GPT slot with its next request, diagnostics retains +`requestedSlotSizes`: the configured `AuctionSlot.formats` list Trusted Server supplied +to GPT for that request. It is a bounded validated copy of the complete configured +list, not an inferred responsive size or a claim about the final selected size. It is +omitted for publisher and otherwise unknown request paths where Trusted Server did not +supply formats. + +For an explicitly filled render, diagnostics can also retain `observedSlotSize`: the +current outer CSS box of the uniquely bound, connected slot element. This is measured +after `slotRenderEnded`. When `ResizeObserver` is available, it remains current +while that same request cycle is latest for the GPT slot; otherwise it is the most +recently sampled box. It is displayed separately from `size`, which remains the exact +GPT-reported `slotRenderEnded.size` fill-size fact. The panel and badge label the three +separate facts as requested slot sizes, GPT-reported fill size, and observed outer slot +box. The observed box may differ from GPT's reported size (for example, a flexible APS +creative can report `1×1` while its allocated outer slot box is larger). It is a +publisher-page layout measurement, not a claim about universal internal creative-pixel +dimensions. Empty, unbound, missing, or ambiguous slots do not report an observed box; +delayed measurements from an older cycle are rejected after a refresh. + +Cross-origin and SafeFrame boundaries prevent diagnostics from inspecting iframe +content. It does not inspect iframe content or alter the APS sandbox, so it cannot +use this field to prove the inner creative's pixels. + Badges and the panel live in a closed Shadow DOM. Diagnostics do not add attributes, classes, or inline styles to publisher slot elements. @@ -326,6 +411,8 @@ The allowlisted export contains: - `version: 1` and an ISO `capturedAt` timestamp. - Current page origin and pathname, excluding query parameters and fragments. - Retained slots, binding facts, visibility, and request cycles. +- `requestedSlotSizes` when Trusted Server supplied configured formats for that exact + request, plus GPT-reported fill `size` and an optional observed outer `observedSlotSize`. - Request path, request intent ID, opportunity, creative-progress timestamps, and safe failure enums. - The per-auction diagnostics token (`trustedServerAuctionId`) and the @@ -340,10 +427,13 @@ The allowlisted export contains: It does not contain raw targeting, bid IDs, bid prices, bidder identity, creative markup, cache URLs, cache payloads, cache or bridge error details, cookies, user -identifiers, query strings, or URL fragments. The exported -`trustedServerAuctionId` is a token minted fresh for each server-side auction: it -is not derived from the Edge Cookie ID or any other visitor identifier, and it does -not repeat across auctions, so it cannot be joined back to a visitor. +identifiers, query strings, or URL fragments. The exported `trustedServerAuctionId` +is the `hb_auction_id` value described in +[Auction correlation token](#auction-correlation-token): minted fresh for each +server-side auction, not derived from the Edge Cookie ID or any other visitor +identifier, and never repeated across auctions, so it cannot be joined back to a +visitor. Diagnostics retain it only after trimming to a non-empty value of at most +256 UTF-8 bytes. Captured records are memory-only. Diagnostics do not add an upload, diagnostics network request, `localStorage`, `sessionStorage`, IndexedDB, or other persistence. @@ -362,6 +452,7 @@ inaccessible to JavaScript. - Retained request cycles per slot: 10. - Retained callback issues: 128. - Retained auction-slot-to-GPT-slot associations: 64. +- Requested slot sizes per correlated request: 16 valid positive sizes. - Retained creative attempts, including status tombstones: 128. - Retained attribution issues: 128. @@ -419,6 +510,56 @@ documented callbacks do not expose a request-cycle identifier. Avoid overlap in controlled tests, or use the issue record as evidence that correlation was not possible. +### A refresh is `unattributed` or `competing` + +`unattributed` means no request-path evidence was still eligible when GPT emitted +`slotRequested`. Each source's marker lives five seconds and is consumed once, so a +request more than five seconds after the observation, a refresh function reference the +publisher captured before installation, and any path Trusted Server does not observe +all stay `unattributed`. Diagnostics never fill that gap from timing, element IDs, or +targeting names. + +`competing` means two or more sources contributed evidence for the same request. It is +a warning that competition or overwrite is possible, not a statement about which values +GPT sent. To narrow it in a controlled test, trigger one path at a time and leave more +than five seconds between refreshes. + +### Delivery stays `candidate_unconfirmed` + +The cycle rendered explicitly non-empty with a Trusted Server candidate, but no matched +creative markup request arrived within five seconds of `slotRenderEnded`. Read the +cycle's other facts before concluding anything: + +- `responseClass` and the GAM identifiers show what Ad Manager reported delivering. +- A creative-bridge failure category on the same cycle shows the bridge was reached and + failed. +- An attribution issue at the same time shows the request arrived but could not be + correlated. +- No evidence at all is consistent with a different GAM result, a targeting overwrite, + and a PUC configuration or ID mismatch alike. + +A late positive observation upgrades the state, so re-read the panel rather than +exporting immediately after render. + +### Correlation evidence is missing + +Attribution issues record why creative evidence could not be attached. They never +increment callback coverage and never produce a delivery claim: + +| Reason | What was observed | +| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `creative_request_without_slot` | The markup request carried no auction slot ID, or no retained association mapped it to a GPT slot. | +| `creative_request_without_cycle` | The slot had no retained request cycle inside the 30-second attempt window, or its most recent cycle was already reported empty. | +| `creative_request_ambiguous_cycle` | An earlier non-empty cycle for the same slot was still in window before render, so no cycle was chosen. | +| `creative_request_on_empty_cycle` | GPT later reported the matched cycle empty, so the attempt was dropped instead of claiming delivery. | +| `creative_attempt_capacity` | The 128-attempt bound was reached with every retained attempt still live. | +| `creative_attempt_unknown` | A response or failure referenced an attempt no longer retained. | +| `creative_attempt_expired` | The attempt passed its 30-second lifetime before its response or failure was observed. | +| `creative_attempt_evicted` | The attempt's slot or request cycle was dropped first, by a retention bound or by GPT reporting that cycle empty. | + +Repeated issues on a busy page usually mean retention bounds, not delivery failure. +Reduce refresh overlap or capture a shorter session, then re-read the cycle. + ## Limits The integration observes six documented PubAdsService events, wraps diff --git a/docs/guide/integrations/gpt.md b/docs/guide/integrations/gpt.md index f38f68231..2093e2cb1 100644 --- a/docs/guide/integrations/gpt.md +++ b/docs/guide/integrations/gpt.md @@ -53,6 +53,7 @@ Add GPT configuration to `trusted-server.toml`: ```toml [integrations.gpt] enabled = true +gam_attribution_enabled = false script_url = "https://securepubads.g.doubleclick.net/tag/js/gpt.js" cache_ttl_seconds = 3600 rewrite_script = true @@ -60,12 +61,18 @@ rewrite_script = true ### Configuration Options -| Field | Type | Required | Default | Description | -| ------------------- | ------- | -------- | ------------------------------------------------------ | ------------------------------------------ | -| `enabled` | boolean | No | `true` | Enable/disable the integration | -| `script_url` | string | No | `https://securepubads.g.doubleclick.net/tag/js/gpt.js` | URL for the GPT bootstrap script | -| `cache_ttl_seconds` | integer | No | `3600` | Cache TTL for proxied scripts (60--86400s) | -| `rewrite_script` | boolean | No | `true` | Whether to rewrite GPT script URLs in HTML | +| Field | Type | Required | Default | Description | +| ------------------------- | ------- | -------- | ------------------------------------------------------ | ----------------------------------------------------------------- | +| `enabled` | boolean | No | `true` | Enable/disable the integration | +| `gam_attribution_enabled` | boolean | No | `false` | Add fixed page-level `ts=true` targeting for GAM cohort reporting | +| `script_url` | string | No | `https://securepubads.g.doubleclick.net/tag/js/gpt.js` | URL for the GPT bootstrap script | +| `cache_ttl_seconds` | integer | No | `3600` | Cache TTL for proxied scripts (60--86400s) | +| `rewrite_script` | boolean | No | `true` | Whether to rewrite GPT script URLs in HTML | + +The environment override +`TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED` works only when +`gam_attribution_enabled` is already present under `[integrations.gpt]` in the +TOML file. EdgeZero v0.0.4 cannot create a missing configuration leaf. ## Endpoints @@ -109,6 +116,65 @@ Takes over `googletag.cmd` so every queued callback is wrapped before GPT execut - Consent gating of ad requests - Ad-unit path rewriting for A/B testing +### GAM Treatment Attribution + +Setting `gam_attribution_enabled = true` adds the fixed page-level GPT targeting +value `ts=true`. It is applied before publisher GPT initialization and remains +for the browser document's lifetime, so initial, lazy, refresh, publisher-owned, +and SPA-route requests inherit it unless another targeting consumer clears or +overrides the key. The attribution switch is independently controlled and +defaults to `false`, but the GPT integration's `enabled` master switch must also +be `true`. + +This key is distinct from the existing slot-level `ts_initial=1` value. +`ts_initial` retains its current cleanup lifecycle; Trusted Server does not +clear the page-level `ts` value during Prebid refresh or SPA cleanup. + +For an eligible publisher document whose activation script was not cloned, +`ts=true` means Trusted Server emitted the rewritten document head before the +GPT request. It does not prove that the response body completed, that a Trusted +Server bid won, or that an impression was caused by treatment. A publisher can +copy the activation script with `srcdoc` or `document.write`; treat any marker +on an unrewritten nested document as contamination, not attribution proof. + +Before enabling attribution in a cohort: + +1. Complete privacy and CSP review, create the reportable predefined `true` + value in the target GAM network, and verify the chosen GAM reporting surface + and billing approval. +2. Audit the short `ts` key across publisher GPT code, effective Prebid + `bidderSettings[*].adserverTargeting` output (including + `setTargetingForGPTAsync`), the effective creative-opportunity targeting map, + and every GAM consumer that can affect eligibility, pricing, protection, or + routing. Trusted Server accepts and forwards operator targeting verbatim; it + does not reserve, filter, or intercept a slot-level `ts` key at runtime. +3. With treatment routing stopped, deploy attribution enabled and validate + initial, lazy, refresh, publisher-owned, and SPA requests. Confirm every + excluded path reports zero marked requests, then save a short paired-report + dry run that satisfies the invariants below before starting the cohort. + +For reporting, save one exact eligible universe: GAM network, inventory units, +routes, formats, time zone, date window, metrics, and all exclusions. Report A +is the nonduplicated total for that universe. Report B uses identical filters +and metrics plus exactly `ts=true`. If Enhanced Key-Value reporting is +unavailable, unapproved, or incompatible, use an exactly filtered legacy +key-value report and never sum its repeated key-value rows. Derive control as +`A - B`, and require `0 <= B <= A` for every metric. A violation invalidates the +whole report pair; never clamp a negative result. Use the same reporting-latency +and invalid-traffic maturation window for both reports. + +GAM results are descriptive delivery attribution, not a causal treatment +effect. Aggregate monitoring and synthetic/manual samples can detect obvious +failures but cannot prove marker completeness on every production request +without request-correlated telemetry. + +For a normal rollback, first stop and verify new treatment assignment at the +router, record a clean reporting boundary, and let already-open documents drain. +Exclude the drain interval, then set `gam_attribution_enabled = false` after +marked traffic reaches zero for the agreed interval. An emergency kill may flip +the setting immediately, but the affected interval and subsequent drain must be +treated as invalid for experiment reporting. + ## Use Cases ### First-Party Ad Delivery @@ -144,6 +210,7 @@ Takes over `googletag.cmd` so every queued callback is wrapped before GPT execut - Check the proxy responses have `200` status (look for `X-GPT-Proxy: true` header) - Verify the `script_url` config points to the correct GPT endpoint - Review server logs for upstream fetch failures +- Open [GPT Runtime Diagnostics](./gpt-diagnostics.md) with `?ts_console=1` to see the observed request, render, load, and delivery evidence per slot ## Implementation @@ -152,6 +219,7 @@ Takes over `googletag.cmd` so every queued callback is wrapped before GPT execut ## Next Steps +- Use [GPT Runtime Diagnostics](/guide/integrations/gpt-diagnostics) to inspect GPT lifecycle and Trusted Server delivery evidence in the browser - Review [Integrations Overview](/guide/integrations-overview) for comparison with other integrations - Check [Configuration Reference](/guide/configuration) for advanced options - Learn about [First-Party Proxy](/guide/first-party-proxy) architecture diff --git a/docs/package-lock.json b/docs/package-lock.json index e31c9db5e..e6003067a 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -14,7 +14,7 @@ "eslint": "^10", "mermaid": "^11.12.3", "prettier": "^3.4.2", - "typescript-eslint": "8.57", + "typescript-eslint": "8.66", "vitepress": "^1.5.0", "vitepress-plugin-mermaid": "^2.0.17" } @@ -180,7 +180,6 @@ "integrity": "sha512-Jc360x4yqb3eEg4OY4KEIdGePBxZogivKI+OGIU8aLXgAYPTECvzeOBc90312yHA1hr3AeRlAFl0rIc8lQaIrQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@algolia/client-common": "5.50.0", "@algolia/requester-browser-xhr": "5.50.0", @@ -1886,7 +1885,6 @@ "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -1914,20 +1912,20 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.57.2.tgz", - "integrity": "sha512-NZZgp0Fm2IkD+La5PR81sd+g+8oS6JwJje+aRWsDocxHkjyRw0J5L5ZTlN3LI1LlOcGL7ph3eaIUmTXMIjLk0w==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.66.0.tgz", + "integrity": "sha512-p088eaGrzYz1s+7cov0aMOCkNGTJlVxF4jgubf28c8L0Cv9Rloj8YBHnv4hXLq6IIEE1AsjNWavO+k+8kP2Y0A==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/type-utils": "8.57.2", - "@typescript-eslint/utils": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/scope-manager": "8.66.0", + "@typescript-eslint/type-utils": "8.66.0", + "@typescript-eslint/utils": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1937,15 +1935,15 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.57.2", + "@typescript-eslint/parser": "^8.66.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", - "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz", + "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==", "dev": true, "license": "MIT", "engines": { @@ -1953,17 +1951,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.57.2.tgz", - "integrity": "sha512-30ScMRHIAD33JJQkgfGW1t8CURZtjc2JpTrq5n2HFhOefbAhb7ucc7xJwdWcrEtqUIYJ73Nybpsggii6GtAHjA==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.66.0.tgz", + "integrity": "sha512-X6ypGChaWYk6PBtUg2BwuTZEFFcHJAtGTVJ9/lCTOufhZ4i9fNolQNnktq+kkMCwMj7V8Svsq7+TxSDslmhE0g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/scope-manager": "8.66.0", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0", "debug": "^4.4.3" }, "engines": { @@ -1975,18 +1972,18 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.57.2.tgz", - "integrity": "sha512-FuH0wipFywXRTHf+bTTjNyuNQQsQC3qh/dYzaM4I4W0jrCqjCVuUh99+xd9KamUfmCGPvbO8NDngo/vsnNVqgw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.66.0.tgz", + "integrity": "sha512-7MthGPTt4BP69lSryqpqq8HQqxuzynssckL/jyDyk3+TNMQ3y2jFWkptCrktWvBrP+EH787Nl5N5Qpw7WZg+5g==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.57.2", - "@typescript-eslint/types": "^8.57.2", + "@typescript-eslint/tsconfig-utils": "^8.66.0", + "@typescript-eslint/types": "^8.66.0", "debug": "^4.4.3" }, "engines": { @@ -1997,18 +1994,18 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.57.2.tgz", - "integrity": "sha512-snZKH+W4WbWkrBqj4gUNRIGb/jipDW3qMqVJ4C9rzdFc+wLwruxk+2a5D+uoFcKPAqyqEnSb4l2ULuZf95eSkw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.66.0.tgz", + "integrity": "sha512-8TGcH25j9zqJ/IULB/ppyhRvxA8QYfFEZ7nfbg6/BN9spDgb8fPWQXlE5l8TWBL50EtUx007uZ1o9VOwrq2/9g==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2" + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2019,9 +2016,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.57.2.tgz", - "integrity": "sha512-3Lm5DSM+DCowsUOJC+YqHHnKEfFh5CoGkj5Z31NQSNF4l5wdOwqGn99wmwN/LImhfY3KJnmordBq/4+VDe2eKw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.66.0.tgz", + "integrity": "sha512-9D5gLYZG4rOjcoag8MQ/fWI8WqA9wcPDyOGyWtWFhvM1lHRbliqUSPIY5J3zqCU1tvSwzXxnnjhQhz5Ne7mJ4g==", "dev": true, "license": "MIT", "engines": { @@ -2032,21 +2029,21 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.57.2.tgz", - "integrity": "sha512-Co6ZCShm6kIbAM/s+oYVpKFfW7LBc6FXoPXjTRQ449PPNBY8U0KZXuevz5IFuuUj2H9ss40atTaf9dlGLzbWZg==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.66.0.tgz", + "integrity": "sha512-LG2dWfjZQQp0ADtAu/EWJVayefGL2UEZ3CDeI44D9v3rXB/WYUqE/jpO28KrEKul5AySrmI+Zh1v6v+xW2U9+g==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/utils": "8.57.2", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0", + "@typescript-eslint/utils": "8.66.0", "debug": "^4.4.3", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2057,13 +2054,13 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.57.2.tgz", - "integrity": "sha512-/iZM6FnM4tnx9csuTxspMW4BOSegshwX5oBDznJ7S4WggL7Vczz5d2W11ecc4vRrQMQHXRSxzrCsyG5EsPPTbA==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.66.0.tgz", + "integrity": "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ==", "dev": true, "license": "MIT", "engines": { @@ -2075,21 +2072,21 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.57.2.tgz", - "integrity": "sha512-2MKM+I6g8tJxfSmFKOnHv2t8Sk3T6rF20A1Puk0svLK+uVapDZB/4pfAeB7nE83uAZrU6OxW+HmOd5wHVdXwXA==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.66.0.tgz", + "integrity": "sha512-8/x4INiiQb10jGgXYD7116/zQ+OL84ZIFn0za68wwFHCanT/VLbBEroWht8RV8fn0/ZCAoazHLQgwUC0UQcDfg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.57.2", - "@typescript-eslint/tsconfig-utils": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/project-service": "8.66.0", + "@typescript-eslint/tsconfig-utils": "8.66.0", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2099,20 +2096,20 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.57.2.tgz", - "integrity": "sha512-krRIbvPK1ju1WBKIefiX+bngPs+odIQUtR7kymzPfo1POVw3jlF+nLkmexdSSd4UCbDcQn+wMBATOOmpBbqgKg==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.66.0.tgz", + "integrity": "sha512-jasearZPolBw5NJNYGMwxzHMF83niVWmMU1VdHzG1CyfI2VS7f7nZltnKtHcg20hW+7Uo5GfK4MeDPoU3qI8EA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2" + "@typescript-eslint/scope-manager": "8.66.0", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2123,17 +2120,17 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.57.2.tgz", - "integrity": "sha512-zhahknjobV2FiD6Ee9iLbS7OV9zi10rG26odsQdfBO/hjSzUQbkIYgda+iNKK1zNiW2ey+Lf8MU5btN17V3dUw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.66.0.tgz", + "integrity": "sha512-dkKR8q+lKciskj1Y3vthHktl+3cMLWGyVUP23bRiPZ5O9BRT++4EqDDV+TVeIKBL1VXVEqrJlz8MYbcnvJcAlg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", + "@typescript-eslint/types": "8.66.0", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -2433,7 +2430,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2474,7 +2470,6 @@ "integrity": "sha512-yE5I83Q2s8euVou8Y3feXK08wyZInJWLYXgWO6Xti9jBUEZAGUahyeQ7wSZWkifLWVnQVKEz5RAmBlXG5nqxog==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@algolia/abtesting": "1.16.0", "@algolia/client-abtesting": "5.50.0", @@ -2567,7 +2562,6 @@ "integrity": "sha512-opLQzEVriiH1uUQ4Kctsd49bRoFDXGGSC4GUqj7pGyxM3RehRhvTlZJc1FL/Flew2p5uwxa1tUDWKzI4wNM8pg==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "@chevrotain/cst-dts-gen": "11.1.2", "@chevrotain/gast": "11.1.2", @@ -2672,7 +2666,6 @@ "integrity": "sha512-iJc4TwyANnOGR1OmWhsS9ayRS3s+XQ185FmuHObThD+5AeJCakAAbWv8KimMTt08xCCLNgneQwFp+JRJOr9qGQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10" } @@ -3107,7 +3100,6 @@ "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "dev": true, "license": "ISC", - "peer": true, "engines": { "node": ">=12" } @@ -3363,7 +3355,6 @@ "integrity": "sha512-S9jlY/ELKEUwwQnqWDO+f+m6sercqOPSqXM5Go94l7DOmxHVDgmSFGWEzeE/gwgTAr0W103BWt0QLe/7mabIvA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", @@ -3613,7 +3604,6 @@ "integrity": "sha512-/yNdlIkpWbM0ptxno3ONTuf+2g318kh2ez3KSeZN5dZ8YC6AAmgeWz+GasYYiBJPFaYcSAPeu4GfhUaChzIJXA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "tabbable": "^6.4.0" } @@ -3979,7 +3969,6 @@ "integrity": "sha512-fEnci+Immw6lKMFI8sqzjlATTyjLkRa6axrEgLV2yHTfv8r+h1wjFbV6xeRtd4rUV1cS4EpR9rwp3Rci7TRWDw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@braintree/sanitize-url": "^7.1.1", "@iconify/utils": "^3.0.2", @@ -4300,12 +4289,11 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -4552,9 +4540,9 @@ "peer": true }, "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "dev": true, "license": "ISC", "bin": { @@ -4688,14 +4676,14 @@ } }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" @@ -4767,16 +4755,16 @@ } }, "node_modules/typescript-eslint": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.57.2.tgz", - "integrity": "sha512-VEPQ0iPgWO/sBaZOU1xo4nuNdODVOajPnTIbog2GKYr31nIlZ0fWPoCQgGfF3ETyBl1vn63F/p50Um9Z4J8O8A==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.66.0.tgz", + "integrity": "sha512-QlEbBPz/RuJ1XUHj29nm3t0F/O/cSlEnntozqPOYHnnTGAXFamnMBu5i9Vn6vhUPHGAjR+Vl+5J8vPN/BMUrJw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.57.2", - "@typescript-eslint/parser": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/utils": "8.57.2" + "@typescript-eslint/eslint-plugin": "8.66.0", + "@typescript-eslint/parser": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0", + "@typescript-eslint/utils": "8.66.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -4787,7 +4775,7 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/ufo": { @@ -4937,7 +4925,6 @@ "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", @@ -4998,7 +4985,6 @@ "integrity": "sha512-+2ym1/+0VVrbhNyRoFFesVvBvHAVMZMK0rw60E3X/5349M1GuVdKeazuksqopEdvkKwKGs21Q729jX81/bkBJg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@docsearch/css": "3.8.2", "@docsearch/js": "3.8.2", @@ -5110,7 +5096,6 @@ "integrity": "sha512-iV/sU9SzOlmA/0tygSmjkEN6Jbs3nPoIPFhCMLD2STrjgOU8DX7ZtzMhg4ahVwf5Rp9KoFzcXeB1ZrVbLBp5/Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vue/compiler-dom": "3.5.31", "@vue/compiler-sfc": "3.5.31", diff --git a/docs/package.json b/docs/package.json index f74991a4c..df1bd66d0 100644 --- a/docs/package.json +++ b/docs/package.json @@ -20,7 +20,7 @@ "eslint": "^10", "mermaid": "^11.12.3", "prettier": "^3.4.2", - "typescript-eslint": "8.57", + "typescript-eslint": "8.66", "vitepress": "^1.5.0", "vitepress-plugin-mermaid": "^2.0.17" } diff --git a/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md b/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md new file mode 100644 index 000000000..f6dbab2b5 --- /dev/null +++ b/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md @@ -0,0 +1,2141 @@ +# Server-Side Ad Template CLI Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the unified `ts` CLI support for server-side ad-template static diagnostics and browser-backed verification described in `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md`. + +**Architecture:** Keep the CLI host-only and thin: Clap parsing stays in `run.rs` / command adapter modules, shared app-config loading moves to `app_config.rs`, pure ad-template logic lives under `ad_templates/`, and Chrome/Chromium collection lives under `audit/`. Runtime gate rules are extracted into a small pure helper in `trusted-server-core` so the CLI does not duplicate server behavior. + +**Tech Stack:** Rust 2024 workspace, host-target `trusted-server-cli`, `clap`, EdgeZero typed app-config loader, `serde`/`serde_json` for stable JSON, `chromiumoxide` for browser-backed audit collection, local HTML fixture tests, and existing `trusted-server-core::creative_opportunities` matching. + +--- + +## Current State + +- Branch: `feature/ts-cli-ad-templates`. +- Static ad-template commands already exist in `crates/trusted-server-cli/src/config_ad_templates.rs`. +- The current branch does not contain #800 audit files. Port useful #800 pieces into the current #799 code shape; do not resurrect stale `args.rs` or `config_command.rs`. +- The spec was updated after review and is the source of truth: + `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md`. +- Keep `.env` and operator-owned `trusted-server.toml` out of commits. + +## File Map + +### New files + +| File | Responsibility | +| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-cli/src/app_config.rs` | Shared effective app-config loader and shared `AppConfigArgs`. | +| `crates/trusted-server-cli/src/ad_templates/mod.rs` | Re-export focused ad-template CLI modules. | +| `crates/trusted-server-cli/src/ad_templates/expected.rs` | Path/URL normalization and expected-slot projection from runtime slot matching. | +| `crates/trusted-server-cli/src/ad_templates/compare.rs` | Pure DOM/GPT/APS evidence comparison, statuses, warnings, runtime gate output, strict failure decisions. | +| `crates/trusted-server-cli/src/ad_templates/output.rs` | Human and JSON rendering for static diagnostics and browser verification. | +| `crates/trusted-server-cli/src/audit/mod.rs` | Audit namespace entry point. | +| `crates/trusted-server-cli/src/audit/page.rs` | Generic page audit command ported from #800. | +| `crates/trusted-server-cli/src/audit/collector.rs` | Browser collector trait plus collected page/evidence structs. | +| `crates/trusted-server-cli/src/audit/browser.rs` | Chromiumoxide-backed browser collector, init scripts, optional scroll, page-level collection errors. | +| `crates/trusted-server-cli/src/audit/ad_templates.rs` | `ts audit ad-templates verify` orchestration. | +| `crates/trusted-server-cli/src/audit/ad_template_collector.js` | Read-only init script for GPT/APS/DOM evidence collection, included via `include_str!`. | + +### Modified files + +| File | Change summary | +| ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| `Cargo.toml` | Add workspace dependencies missing from this branch: `chromiumoxide`, `serde`, and `serde_json` if not present. | +| `crates/trusted-server-cli/Cargo.toml` | Add host-only CLI dependencies for browser audit and JSON output. | +| `crates/trusted-server-cli/src/lib.rs` | Register new `app_config`, `ad_templates`, and `audit` modules under `cfg(not(target_arch = "wasm32"))`. | +| `crates/trusted-server-cli/src/run.rs` | Add `Audit` command namespace, parser tests, and dispatch. | +| `crates/trusted-server-cli/src/config_ad_templates.rs` | Shrink to Clap adapter using shared loader/expected/output modules. | +| `crates/trusted-server-core/src/creative_opportunities.rs` | Add pure runtime gate helper types/functions shared by runtime and CLI. | +| `crates/trusted-server-core/src/publisher.rs` | Route existing server-side ad-stack gate through the shared helper without changing behavior. | + +## Implementation Rules + +- Use TDD for each task: write a failing test first, run it, implement the minimal code, re-run, then commit. +- Commit after each task using repo style: sentence case, imperative, no semantic prefix. +- Keep `trusted-server-cli` host-only. Do not introduce `tokio`, `chromiumoxide`, or filesystem/browser dependencies into core runtime or wasm adapter crates. +- Do not write real publisher domains or secrets in tests. Use `example.com`, `publisher.example`, and fictional IDs only. +- Prefer pure module tests over browser tests. Browser-backed fixture tests should use local HTML only and no GPT/APS network. + +## Task 0: Baseline And Branch Hygiene + +**Files:** + +- Verify: `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` +- Verify: `docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md` + +- [ ] **Step 1: Confirm branch and working tree** + + Run: + + ```bash + git status --short --branch + git log --oneline --decorate -5 + ``` + + Expected: on `feature/ts-cli-ad-templates`; no unrelated modified files besides the approved spec/plan docs. + +- [ ] **Step 2: Run docs format check before code work** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: `All matched files use Prettier code style!` + +- [ ] **Step 3: Commit reviewed spec and plan** + + Run: + + ```bash + git add docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md + git commit -m "Add server-side ad-template CLI implementation plan" + ``` + + Expected: docs-only commit. If the spec commit already exists separately, commit only the plan. + +## Task 1: Share Runtime Ad-Stack Gate Logic + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Write failing core tests for the shared gate helper** + + Add tests near the existing `creative_opportunities` tests: + + ```rust + #[test] + fn ad_stack_gate_passes_for_eligible_navigation() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Yes); + assert!(result.blocking_gates().is_empty()); + } + + #[test] + fn ad_stack_gate_blocks_known_kill_switch() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: false, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::No); + assert!(result.blocking_gates().contains(&AdStackGateName::AuctionEnabled)); + } + + #[test] + fn ad_stack_gate_is_unknown_when_consent_is_unknown() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: None, + auction_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Unknown); + } + + // Locks the spec §5.2 mirror invariant: with Some(consent) supplied for every + // input combination, `expected == Yes` must equal the legacy all-AND boolean. + #[test] + fn ad_stack_gate_with_known_consent_matches_legacy_boolean() { + for bits in 0u8..64 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: Some(bits & 32 != 0), + auction_enabled: bits & 1 == 0, + }; + // Legacy semantics: all positive gates true, both negative gates false. + let legacy = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.consent_allows_auction == Some(true) + && input.auction_enabled; + let got = evaluate_ad_stack_gate(input).expected == RuntimeAdStackExpected::Yes; + assert_eq!(got, legacy, "gate mismatch for bits={bits}"); + } + } + ``` + +- [ ] **Step 2: Run the focused test and verify it fails** + + Run: + + ```bash + # NOTE: trusted-server-core links the `fastly` crate and CANNOT build for the host + # triple — run core tests on the DEFAULT target (wasm32-wasip1 + viceroy runner), + # i.e. no `--target`. Only the host-only `trusted-server-cli` uses `--target `. + cargo test -p trusted-server-core creative_opportunities::tests::ad_stack_gate + ``` + + Expected: compile failure because `AdStackGateInput` / `evaluate_ad_stack_gate` do not exist. + +- [ ] **Step 3: Implement pure gate types and helper** + + Add public, serde-free types to `creative_opportunities.rs`: + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq)] + pub enum RuntimeAdStackExpected { + Yes, + No, + Unknown, + } + + #[derive(Debug, Clone, Copy, Eq, PartialEq)] + pub enum AdStackGateName { + MethodGet, + Navigation, + NotPrefetch, + NotBot, + MatchedSlots, + ConsentAllowsAuction, + AuctionEnabled, + } + + #[derive(Debug, Clone, Copy)] + pub struct AdStackGateInput { + pub method_get: bool, + pub navigation: bool, + pub prefetch: bool, + pub bot: bool, + pub matched_slots: bool, + pub consent_allows_auction: Option, + pub auction_enabled: bool, + } + + #[derive(Debug, Clone, Eq, PartialEq)] + pub struct AdStackGateResult { + pub expected: RuntimeAdStackExpected, + blocking_gates: Vec, + } + + impl AdStackGateResult { + pub fn blocking_gates(&self) -> &[AdStackGateName] { + &self.blocking_gates + } + } + ``` + + Implement `evaluate_ad_stack_gate(input)` so any known blocking boolean gate returns `No`, all known pass plus `Some(true)` consent returns `Yes`, and all known pass plus `None` consent returns `Unknown`. + + Mind the gate polarity, mirroring `should_run_server_side_ad_stack`: `method_get`, + `navigation`, `matched_slots`, and `auction_enabled` block when **false**, while + `prefetch` and `bot` block when **true** (their gate names `NotPrefetch` / `NotBot` + pass when the input bool is false). `consent_allows_auction` is the only tri-state + input: `Some(false)` blocks (No), `Some(true)` passes, `None` yields Unknown only + when no other gate already blocks. + +- [ ] **Step 4: Route `publisher.rs` through the helper** + + Replace the body of `should_run_server_side_ad_stack` with a call to `evaluate_ad_stack_gate`, preserving the existing function signature for low-risk runtime compatibility: + + ```rust + crate::creative_opportunities::evaluate_ad_stack_gate( + crate::creative_opportunities::AdStackGateInput { + method_get: is_get, + navigation: is_navigation, + prefetch: is_prefetch, + bot: is_bot, + matched_slots: has_matched_slots, + consent_allows_auction: Some(consent_allows_auction), + auction_enabled, + }, + ) + .expected + == crate::creative_opportunities::RuntimeAdStackExpected::Yes + ``` + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + # Core tests run on the default wasm target via viceroy (no --target). + cargo test -p trusted-server-core publisher::tests + cargo test -p trusted-server-core creative_opportunities + ``` + + Expected: all focused tests pass (including the existing `should_run_server_side_ad_stack` truth-table tests in `publisher::tests`). + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/publisher.rs + git commit -m "Share server-side ad stack gate evaluation" + ``` + +## Task 2: Extract Shared CLI App Config Loader + +**Files:** + +- Create: `crates/trusted-server-cli/src/app_config.rs` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` + +- [ ] **Step 1: Write failing loader tests** + + Move the existing temp-project helpers from `config_ad_templates.rs` tests into `app_config.rs` tests and add: + + ```rust + #[test] + fn explicit_missing_app_config_does_not_fall_back() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let missing_path = temp.path().join("missing.toml"); + + let args = AppConfigArgs { + app_config: Some(missing_path.clone()), + manifest: manifest_path, + no_env: true, + }; + + let err = load_settings(&args).expect_err("should reject missing explicit config"); + assert!( + err.contains(missing_path.to_string_lossy().as_ref()), + "error should mention the explicit missing path" + ); + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli app_config --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because `app_config` module is not registered. + +- [ ] **Step 3: Implement `app_config.rs`** + + Move these items out of `config_ad_templates.rs`: + - `AppConfigArgs` + - `LoadedSettings` + - `load_settings` + - `resolve_app_config_path` + + Make the API explicit: + + ```rust + #[derive(Clone, Debug, Args)] + pub struct AppConfigArgs { + #[arg(long)] + pub app_config: Option, + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + #[arg(long)] + pub no_env: bool, + } + + pub struct LoadedSettings { + pub app_config_path: PathBuf, + pub settings: Settings, + } + + pub fn load_settings(args: &AppConfigArgs) -> Result { + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + let app_config_path = + resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); + + let mut opts = AppConfigLoadOptions::default(); + opts.env_overlay = !args.no_env; + let app_config = app_config::deserialize_app_config_with_options::( + &app_config_path, + &app_name, + &opts, + ) + .map_err(|err| format!("failed to load {}: {err}", app_config_path.display()))?; + + Ok(LoadedSettings { + app_config_path, + settings: app_config.into_settings(), + }) + } + + fn resolve_app_config_path( + explicit: Option<&Path>, + manifest_path: &Path, + app_name: &str, + ) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + let file_name = format!("{app_name}.toml"); + if let Some(parent) = manifest_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + parent.join(file_name) + } else { + PathBuf::from(file_name) + } + } + ``` + + Include the same top-level imports currently used by these helpers: + `std::path::{Path, PathBuf}`, `clap::Args`, + `edgezero_core::app_config::{self, AppConfigLoadOptions}`, + `edgezero_core::manifest::ManifestLoader`, + `trusted_server_core::config::TrustedServerAppConfig`, and + `trusted_server_core::settings::Settings`. + +- [ ] **Step 4: Register module and update imports** + + In `lib.rs`, add: + + ```rust + #[cfg(not(target_arch = "wasm32"))] + mod app_config; + ``` + + In `config_ad_templates.rs`, import: + + ```rust + use crate::app_config::{load_settings, AppConfigArgs}; + ``` + +- [ ] **Step 5: Run focused CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli app_config --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: existing static command behavior remains unchanged. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/src/app_config.rs crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/lib.rs + git commit -m "Extract shared CLI app config loader" + ``` + +## Task 3: Add Expected-Slot Model + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/mod.rs` +- Create: `crates/trusted-server-cli/src/ad_templates/expected.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` + +- [ ] **Step 1: Write failing expected-slot tests** + + Add a test-only dependency to `crates/trusted-server-cli/Cargo.toml` so tests can + deserialize core slot config instead of constructing `CreativeOpportunitySlot` with + its `pub(crate)` `compiled_patterns` cache: + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] + toml = { workspace = true } + ``` + + In `expected.rs`, add tests for path normalization, full URL normalization, config-order preservation, resolved div ID, resolved GAM unit path, provider names, and matching page patterns: + + ```rust + fn creative_config_with_slots(patterns: &[&str]) -> CreativeOpportunitiesConfig { + let page_patterns = patterns + .iter() + .map(|pattern| format!("\"{pattern}\"")) + .collect::>() + .join(", "); + let toml = format!( + r#" + gam_network_id = "123" + auction_timeout_ms = 500 + price_granularity = "dense" + + [[slot]] + id = "atf" + gam_unit_path = "/123/news/atf" + div_id = "ad-atf-" + page_patterns = [{page_patterns}] + formats = [{{ width = 300, height = 250 }}] + floor_price = 0.50 + targeting = {{ zone = "atf" }} + + [slot.providers.prebid] + bidders = {{}} + "# + ); + let mut config = toml::from_str::(&toml) + .expect("should deserialize creative opportunities config"); + config.compile_slots(); + config + } + + #[test] + fn expected_slots_use_runtime_matcher_and_config_order() { + let config = creative_config_with_slots(["/news/*", "/"].as_slice()); + let expected = expected_slots_for_path("/news/story", &config) + .expect("should build expected slots"); + + assert_eq!(expected.path, "/news/story"); + assert_eq!(expected.slots.iter().map(|slot| slot.id.as_str()).collect::>(), ["atf"]); + assert_eq!(expected.slots[0].div_id, "ad-atf-"); + assert_eq!(expected.slots[0].gam_unit_path, "/123/news/atf"); + assert_eq!(expected.slots[0].providers, ["prebid"]); + } + + #[test] + fn normalize_path_or_url_strips_query_and_fragment() { + assert_eq!(normalize_path_or_url("https://www.example.com/news/story?x=1#top").expect("should normalize"), "/news/story"); + assert_eq!(normalize_path_or_url("news/story?x=1").expect("should normalize"), "/news/story"); + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::expected --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because module/types do not exist. + +- [ ] **Step 3: Implement expected-slot structs** + + Define pure structs that own strings and are stable for output: + + ```rust + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedSlots { + pub path: String, + pub slots: Vec, + } + + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedSlot { + pub id: String, + pub div_id: String, + pub gam_unit_path: String, + pub formats: Vec, + pub providers: Vec, + pub page_patterns: Vec, + } + + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedFormat { + pub width: u32, + pub height: u32, + // Mirrors `MediaType` rendered as a stable string (`"banner"`, `"video"`, `"native"`). + pub media_type: String, + } + ``` + + `div_id` and `gam_unit_path` are resolved (non-optional) strings. The core + `CreativeOpportunitySlot` stores `div_id` / `gam_unit_path` as `Option` + and the GAM unit path is composed with the configured GAM network ID; mirror the + existing `format_slot` resolution in `config_ad_templates.rs` so the CLI does not + invent a second resolution rule. Use + `trusted_server_core::creative_opportunities::match_slots`. Do not compile globs in CLI. + +- [ ] **Step 4: Register `ad_templates` and update static commands** + + In `lib.rs`, add: + + ```rust + #[cfg(not(target_arch = "wasm32"))] + mod ad_templates; + ``` + + Rewire `config_ad_templates.rs` onto the shared module, and remove the now-duplicated + local code so there is no name collision or dead `normalize_path_or_url`: + - delete the private `fn normalize_path_or_url` (currently `config_ad_templates.rs:448`) + and add `use crate::ad_templates::expected::{expected_slots_for_path, normalize_path_or_url};`; + - the existing `config_ad_templates::tests::normalizes_path_or_url_like_runtime_request_path` + test (currently `:661`) calls the local fn via `super::*` — either delete it (Task 3 + Step 1 already adds normalization tests in `expected.rs`) or repoint it at + `crate::ad_templates::expected::normalize_path_or_url`. Pick one so the test crate + still compiles at this commit. + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::expected --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all pass. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/expected.rs crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/lib.rs + git commit -m "Add shared ad-template expected slot model" + ``` + +## Task 4: Add Stable Output And JSON Types + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/ad_templates/mod.rs` + +- [ ] **Step 1: Add CLI JSON dependencies** + + The CLI crate has **no plain `[dependencies]` table** — every runtime dep lives + under `[target.'cfg(not(target_arch = "wasm32"))'.dependencies]` (the workspace + default build target is `wasm32-wasip1` per `.cargo/config.toml`). Add the new deps + to that existing table; do **not** create a `[dependencies]` table, or they compile + for wasm and leak host-only crates into the wasm build: + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] + # ... existing clap/url/etc ... + serde = { workspace = true } + serde_json = { workspace = true } + ``` + + Add workspace dependency `chromiumoxide = "0.9.1"` in `Cargo.toml` only in Task 7 when browser code is introduced. + +- [ ] **Step 2: Write failing JSON output tests** + + In `output.rs`, add tests that construct an in-memory verification result and assert exact JSON values: + + ```rust + #[test] + fn verification_json_contains_gate_state_and_extra_evidence() { + let result = VerificationReport::example_confirmed_with_extra_evidence(); + let value = serde_json::to_value(&result).expect("should serialize"); + + assert_eq!(value["ok"], true); + assert_eq!(value["pages"][0]["requested_path"], "/news/story"); + assert_eq!(value["pages"][0]["runtime_ad_stack_expected"], "unknown"); + assert_eq!(value["pages"][0]["extra_evidence"][0]["kind"], "gpt"); + assert_eq!(value["pages"][0]["warnings"][0]["code"], "redirected"); + } + + // Pins the spec §8 navigation_failed shape: error present, runtime/gates/ + // matched_slot_count keys ABSENT (skipped), final_url/path null. + #[test] + fn page_error_json_matches_navigation_failed_shape() { + let result = VerificationReport::example_navigation_failed(); + let value = serde_json::to_value(&result).expect("should serialize"); + let page = &value["pages"][0]; + + assert_eq!(page["error"]["code"], "navigation_failed"); + assert!(page["final_url"].is_null(), "final_url should be null"); + assert!(page["path"].is_null(), "path should be null"); + assert!(page.get("runtime_ad_stack_expected").is_none(), "runtime field absent on error page"); + assert!(page.get("gates").is_none(), "gates absent on error page"); + assert!(page.get("matched_slot_count").is_none(), "matched_slot_count absent on error page"); + assert_eq!(value["ok"], false); + } + ``` + +- [ ] **Step 3: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::output --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because output model does not exist. + +- [ ] **Step 4: Implement serializable output types** + + Model the **entire** `--json` wire tree from spec §8 (this is the single source of + truth for field names and ordering). Use owned `String` / `Vec` fields and + `#[serde(rename_all = "snake_case")]` so output is stable. Leaf enums: + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum SlotStatus { Confirmed, Partial, Missing } + + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum RuntimeAdStackExpectedJson { Yes, No, Unknown } + + impl From for RuntimeAdStackExpectedJson { + fn from(value: trusted_server_core::creative_opportunities::RuntimeAdStackExpected) -> Self { + use trusted_server_core::creative_opportunities::RuntimeAdStackExpected as Core; + match value { + Core::Yes => Self::Yes, + Core::No => Self::No, + Core::Unknown => Self::Unknown, + } + } + } + + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum GateState { Pass, Fail, Unknown } + ``` + + Top-level tree (field names and nesting must match §8 exactly): + + ```rust + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct VerificationReport { + pub ok: bool, + pub strict: bool, + pub pages: Vec, + pub warnings: Vec, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct PageJson { + pub url: String, + pub final_url: Option, + pub requested_path: String, + pub path: Option, + // Field ORDER matters: serde serializes in declaration order. Spec §8 places + // `error` immediately after `path` on the navigation_failed shape, so it must + // be declared here (not last). On normal pages `error` is None and skipped, so + // the runtime/gates/slots run in §8 order; on error pages the runtime/gates/ + // matched_slot_count are None and skipped, leaving url..path, error, slots, + // extra_evidence, warnings — exactly the §8 error shape. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_ad_stack_expected: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub gates: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_slot_count: Option, + pub slots: Vec, + pub extra_evidence: Vec, + pub warnings: Vec, + } + + // One field per gate name from spec §5.2 / §8, each a GateState. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct Gates { + pub method_get: GateState, + pub navigation: GateState, + pub not_prefetch: GateState, + pub not_bot: GateState, + pub matched_slots: GateState, + pub auction_enabled: GateState, + pub consent_allows_auction: GateState, + } + + // Serialize for output JSON; Deserialize because the browser collector payload + // (Task 8) carries warning objects decoded into `BrowserAdEvidence.warnings`. + #[derive(Debug, Clone, Eq, PartialEq, Serialize, serde::Deserialize)] + pub struct Warning { + pub code: String, + pub message: String, + } + ``` + + Define the remaining nested JSON structs **explicitly** — do not serialize the + compare-module types directly. The compare types (`SlotResult`, `SlotEvidence`, + `GptSlotEvidence`, `ExtraEvidence`) carry a `phase` field and are not `Serialize`; + spec §8's `evidence.gpt` has **no** `phase` key and `configured` excludes `id` + and `page_patterns`. Mismatched reuse would emit extra keys. Wire structs: + + ```rust + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct SlotJson { + pub id: String, + pub status: SlotStatus, + pub phase: EvidencePhaseJson, + pub configured: ConfiguredJson, + pub evidence: SlotEvidenceJson, + pub warnings: Vec, + } + + // §8 `configured`: div_id, gam_unit_path, formats, providers — NO id/page_patterns. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct ConfiguredJson { + pub div_id: String, + pub gam_unit_path: String, + pub formats: Vec, + pub providers: Vec, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct FormatJson { + pub width: u32, + pub height: u32, + pub media_type: String, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct SlotEvidenceJson { + pub dom_id: Option, + pub gpt: Option, + } + + // §8 `evidence.gpt`: gam_unit_path, div_id, sizes — NO phase. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct GptEvidenceJson { + pub gam_unit_path: String, + pub div_id: String, + pub sizes: Vec<[u32; 2]>, + } + + #[derive(Debug, Clone, Copy, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum EvidencePhaseJson { InitialLoad, Scroll } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct ExtraEvidenceJson { + pub kind: String, + pub phase: EvidencePhaseJson, + pub dom_id: Option, + pub gam_unit_path: Option, + pub sizes: Vec<[u32; 2]>, + pub reason: String, + } + ``` + + Note `sizes` serialize as `[[300,250]]` (arrays of two ints), matching §8 — use + `[u32; 2]` here even though the compare module uses `(u32, u32)` tuples; the + Task 9 assembly maps tuple → `[w, h]`. The conversion from the compare + `SlotResult`/`SlotEvidence`/`ExtraEvidence` to these JSON types (dropping `phase` + from `gpt`, dropping `id`/`page_patterns` from `configured`) lives in Task 9 Step 7. + `Warning` is the single warning type for the whole CLI; defined here and re-exported + from `ad_templates::mod` so `compare.rs` reuses it (plain data, not JSON logic). + Keep `example_confirmed_with_extra_evidence()` and similar fixtures behind + `#[cfg(test)]`. + +- [ ] **Step 5: Add verification human-render helpers** + + Add only the **browser-verification** page summary writers here (used by + `audit::ad_templates` in Task 9), writing to `&mut dyn Write`; no `println!` / + `eprintln!`. Do **not** add static match/check/explain writers in this task — + those are the existing `write_match_result`/`format_slot`/etc. functions that + Task 6 Step 3 **moves** out of `config_ad_templates.rs`. Keeping the static + relocation solely in Task 6 avoids two competing copies of the same helpers in + `output.rs`. The verification writers added here may be unused until Task 9 (a + warn-level `dead_code` lint that does not fail `cargo test`); add + `#[allow(dead_code)]` if clippy is run between Task 4 and Task 9. + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::output --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all pass. + +- [ ] **Step 7: Commit** + + ```bash + git add Cargo.toml crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/output.rs + git commit -m "Add ad-template CLI output models" + ``` + +## Task 5: Add Pure Evidence Comparison + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/compare.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/mod.rs` + +- [ ] **Step 1: Write failing comparison tests** + + Cover every spec status and warning case without launching Chrome. Define small + test constructors so tests do not couple to the full `BrowserAdEvidence` field + list (`page_bids` and `warnings` default empty, evidence items default to + `EvidencePhase::InitialLoad`): + + ```rust + fn dom(id: &str) -> DomEvidence { + DomEvidence { dom_id: id.to_string(), phase: EvidencePhase::InitialLoad } + } + + fn gpt_slot(gam_unit_path: &str, div_id: &str, sizes: &[(u32, u32)]) -> GptSlotEvidence { + GptSlotEvidence { + gam_unit_path: gam_unit_path.to_string(), + div_id: div_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn aps(slot_id: &str, sizes: &[(u32, u32)]) -> ApsFetchBidsEvidence { + ApsFetchBidsEvidence { slot_id: slot_id.to_string(), sizes: sizes.to_vec(), phase: EvidencePhase::InitialLoad } + } + + // Non-banner format helper for the unsupported-format test. + fn expected_slot_video(id: &str, div_id: &str, gam_unit_path: &str) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: gam_unit_path.to_string(), + formats: vec![ExpectedFormat { width: 0, height: 0, media_type: "video".to_string() }], + providers: Vec::new(), + page_patterns: Vec::new(), + } + } + + fn evidence(doms: Vec, gpts: Vec, aps: Vec) -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: doms, + gpt_slots: gpts, + aps_calls: aps, + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + #[test] + fn gpt_path_div_and_size_overlap_confirms_slot() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "GPT path+div+size overlap should confirm"); + assert!(result.slots[0].warnings.is_empty(), "confirmed slot should carry no warnings"); + } + + #[test] + fn dom_only_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(vec![dom("ad-atf-0")], Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "DOM-only evidence should be partial"); + assert!( + result.slots[0].warnings.iter().any(|w| w.code == "dom_without_gpt"), + "DOM-only slot should warn dom_without_gpt" + ); + } + + #[test] + fn no_dom_or_gpt_is_missing() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Missing, "no DOM/GPT evidence should be missing"); + } + + #[test] + fn prefix_dom_resolution_ignores_container_suffix() { + let expected = expected_slot("header", "ad-header-0-", "/123/homepage/header", &[(728, 90)], &[]); + // First candidate ends with `-container` and must be skipped; the framework-suffixed ID resolves. + let evidence = evidence( + vec![dom("ad-header-0--container"), dom("ad-header-0-_R_abc123")], + Vec::new(), + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].evidence.dom_id.as_deref(), Some("ad-header-0-_R_abc123"), "prefix match should skip -container"); + assert_eq!(result.slots[0].status, SlotStatus::Partial, "DOM-only prefix match is partial without GPT"); + } + + #[test] + fn unmatched_gpt_slot_becomes_extra_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![ + gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)]), + gpt_slot("/123/publisher/right-rail", "ad-right-rail-0", &[(300, 250)]), + ], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "matched slot still confirms"); + assert_eq!(result.extra_evidence.len(), 1, "unmatched GPT slot becomes extra evidence"); + assert_eq!(result.extra_evidence[0].kind, "gpt"); + assert!(!result.strict_failed(), "extra evidence alone must not fail strict"); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::auction_disabled()); + + assert_eq!(result.runtime_ad_stack_expected, RuntimeAdStackExpected::No, "auction disabled should set No"); + assert_eq!(result.slots[0].status, SlotStatus::Missing, "static status is still reported"); + assert!(!result.strict_failed(), "missing slot must not fail strict when ad stack expected is No"); + } + + // §5.4: GPT path+div match but no numeric size overlap -> partial + warning. + #[test] + fn gpt_incompatible_sizes_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(728, 90)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "no size overlap should be partial"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "incompatible_sizes")); + } + + // §5.4/§5.6: matched slot with only non-banner formats -> partial + unsupported_format. + #[test] + fn non_banner_only_slot_is_partial() { + let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); + let evidence = evidence( + vec![dom("ad-video-0")], + vec![gpt_slot("/123/news/video", "ad-video-0", &[(640, 480)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "non-banner-only should be partial"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "unsupported_format")); + } + + // §5.4: GPT element ID may be `${resolved_dom_id}-container` and still confirm. + #[test] + fn gpt_container_element_id_confirms() { + let expected = expected_slot("atf", "ad-atf-0", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0"), dom("ad-atf-0-container")], + vec![gpt_slot("/123/news/atf", "ad-atf-0-container", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "container element id is a valid GPT div match"); + } + + // §5.4: out-of-page GPT slot is not confirmed; reported as a warning. + #[test] + fn out_of_page_gpt_slot_warns_and_does_not_confirm() { + let expected = expected_slot("interstitial", "ad-oop-", "/123/news/oop", &[(300, 250)], &[]); + // gpt_slot with empty sizes models an out-of-page slot (no numeric sizes). + let evidence = evidence(vec![dom("ad-oop-0")], vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_ne!(result.slots[0].status, SlotStatus::Confirmed, "out-of-page is not confirmed in Phase 1"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "out_of_page_slot")); + } + + // §5.5: matching APS fetchBids -> no provider warning. + #[test] + fn aps_match_adds_no_warning() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + vec![aps("atf", &[(300, 250)])], + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!(!result.slots[0].warnings.iter().any(|w| w.code.starts_with("aps_")), "matching APS should not warn"); + } + + // §5.5: configured aps provider but no APS evidence -> provider warning, still confirmed, strict not failed. + #[test] + fn aps_missing_warns_but_keeps_confirmed() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "missing APS does not flip status"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "aps_evidence_missing")); + assert!(!result.strict_failed(), "provider warning alone must not fail strict"); + } + ``` + + Add a `#[cfg(test)]` constructor in `compare.rs` tests that builds a real + `ExpectedSlot` (the Task 3 type) so comparison tests stay readable: + + ```rust + fn expected_slot(id: &str, div_id: &str, gam_unit_path: &str, sizes: &[(u32, u32)], providers: &[&str]) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: gam_unit_path.to_string(), + formats: sizes.iter().map(|&(width, height)| ExpectedFormat { width, height, media_type: "banner".to_string() }).collect(), + providers: providers.iter().map(|p| p.to_string()).collect(), + page_patterns: Vec::new(), + } + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::compare --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because comparison module does not exist. + +- [ ] **Step 3: Implement browser evidence structs** + + Define the minimum collector-independent input shape plus the comparison result + shape the tests assert against: + + All browser-evidence input structs derive `Debug, Clone` and `serde::Deserialize` + (Task 8 decodes them from the collector's `window.__tsAdTemplateEvidence` JSON); + `EvidencePhase` deserializes from `"initial_load"` / `"scroll"`. The comparison- + result structs derive `Debug` (so the Step 1 `assert_eq!`/`matches!` assertions + compile) and `Clone`. Sizes are `(u32, u32)` tuples internally; deserialize them + from JSON `[w, h]` arrays. + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq, serde::Deserialize)] + #[serde(rename_all = "snake_case")] + pub enum EvidencePhase { + InitialLoad, + Scroll, + } + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct DomEvidence { + pub dom_id: String, + pub phase: EvidencePhase, + } + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct GptSlotEvidence { + pub gam_unit_path: String, + pub div_id: String, + pub sizes: Vec<(u32, u32)>, + pub phase: EvidencePhase, + } + + // APS `apstag.fetchBids` evidence (spec §5.5): configured slot ID + observed sizes. + #[derive(Debug, Clone, serde::Deserialize)] + pub struct ApsFetchBidsEvidence { + pub slot_id: String, + pub sizes: Vec<(u32, u32)>, + pub phase: EvidencePhase, + } + + // DEFERRED in this implementation: `/__ts/page-bids` SPA observation (spec §5.2 + // "when available"). The struct/field are forward scaffolding so the collector and + // JSON can grow it later; Task 8 does NOT populate it and Task 4 JSON does NOT + // surface it in Phase 1. Tracked as a deferred item in Risks. Keep the field so + // `BrowserAdEvidence` deserialization stays forward-compatible (default empty). + #[derive(Debug, Clone, serde::Deserialize)] + pub struct PageBidsEvidence { + pub slot_id: String, + pub phase: EvidencePhase, + } + + // `Warning` is the shared CLI warning type defined in Task 4 (`output.rs`) and + // re-exported from `ad_templates::mod`. It is plain data reused here (not JSON + // logic). Because the collector payload carries warnings, give `Warning` BOTH + // `Serialize` (Task 4 output) and `Deserialize` (Task 8 decode) derives. + use crate::ad_templates::output::Warning; + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct BrowserAdEvidence { + pub dom_ids: Vec, + pub gpt_slots: Vec, + pub aps_calls: Vec, + #[serde(default)] + pub page_bids: Vec, + #[serde(default)] + pub warnings: Vec, + } + + // Comparison output. Uses the core `RuntimeAdStackExpected` enum from Task 1 so + // pure comparison logic does not depend on the output/JSON module. Task 4's + // `RuntimeAdStackExpectedJson` is produced only at serialization time. + #[derive(Debug, Clone)] + pub struct PageVerificationResult { + pub runtime_ad_stack_expected: trusted_server_core::creative_opportunities::RuntimeAdStackExpected, + pub slots: Vec, + pub extra_evidence: Vec, + } + + #[derive(Debug, Clone)] + pub struct SlotResult { + pub id: String, + pub status: SlotStatus, + pub phase: EvidencePhase, + pub evidence: SlotEvidence, + pub warnings: Vec, + } + + #[derive(Debug, Clone)] + pub struct SlotEvidence { + pub dom_id: Option, + pub gpt: Option, + } + + #[derive(Debug, Clone)] + pub struct ExtraEvidence { + pub kind: String, + pub phase: EvidencePhase, + pub dom_id: Option, + pub gam_unit_path: Option, + pub sizes: Vec<(u32, u32)>, + pub reason: String, + } + ``` + + `RuntimeGateSummary` is the third argument to `compare_page_evidence`; it wraps + the core gate result. Provide `RuntimeGateSummary::unknown_allowed()` (expected + `Unknown`) and `RuntimeGateSummary::auction_disabled()` (expected `No`) test + constructors so comparison tests do not rebuild gate inputs by hand. + +- [ ] **Step 4: Implement DOM/GPT/APS rules** + + Status rules: + - DOM exact ID first, then first prefix match, **excluding `-container`** wrappers + (slot-root resolution, spec §5.3). + - GPT confirms when: GAM unit path matches, the GPT slot element ID equals the + resolved DOM ID **or** an existing `${resolved_dom_id}-container` element + (spec §5.4 — note this is the GPT element-ID match, distinct from the §5.3 DOM + root resolution that skips `-container`), and at least one numeric banner size + overlaps. + - GPT path/div match with no numeric size overlap → `partial` (warn `incompatible_sizes`). + - Matched slot whose configured formats are **all non-banner** (video/native) → + `partial` (warn `unsupported_format`); banner is the only Phase-1 confirmable type. + - DOM-only (no GPT) → `partial` (warn `dom_without_gpt`). + - No DOM and no GPT → `missing`. + + Size-compatibility warnings (spec §5.4 — all are warnings, none flip a confirmed + slot to fail): emit a `Warning` for each of: + - `fluid_size_ignored` — non-numeric observed sizes like `"fluid"` ignored for matching; + - `extra_observed_size` — observed GPT sizes not in the configured set; + - `configured_size_not_observed` — configured sizes never observed (when ≥1 was); + - `out_of_page_slot` — out-of-page GPT slot observed; not confirmed in Phase 1. + + Provider + extra evidence: + - APS: configured `providers.aps.slot_id` with matching `fetchBids` → no warning; + missing/ambiguous APS evidence → provider warning only (`aps_evidence_missing` / + `aps_evidence_ambiguous`), never flips status or fails `--strict` in Phase 1. + - Unmatched live DOM/GPT/APS evidence → structured `extra_evidence` (never fails strict). + + Define each warning `code` as a stable string constant so output and tests share them. + +- [ ] **Step 5: Implement strict decision method** + + Add an inherent method on the result so tests can call `result.strict_failed()`: + + ```rust + impl PageVerificationResult { + pub fn strict_failed(&self) -> bool { + use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + if self.runtime_ad_stack_expected == RuntimeAdStackExpected::No { + return false; + } + self.slots + .iter() + .any(|slot| matches!(slot.status, SlotStatus::Missing | SlotStatus::Partial)) + } + } + ``` + + - false when `runtime_ad_stack_expected == No`; + - true for any `missing` or `partial` slot when expected is `Yes` or `Unknown`; + - false for provider warnings and extra evidence alone (they are not slot statuses). + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::compare --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all comparison tests pass. + +- [ ] **Step 7: Commit** + + ```bash + git add crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/compare.rs + git commit -m "Add pure ad-template evidence comparison" + ``` + +## Task 6: Refactor Static Commands Onto Shared Modules + +**Files:** + +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add characterization tests before refactor** + + These guard behavior across the Step 3 move, so each must assert **exact output + substrings** (capture the command's `Vec`/`String` output and + `assert!(out.contains("..."))`), not just run without panicking — a bare + smoke test cannot catch a wording regression. Mirror the existing assertion style + at `config_ad_templates.rs:570-657`. Pin, with concrete expected strings: + - `lint` not configured → e.g. `"creative_opportunities: not configured"`; + - `lint` with slots + auction disabled → slot count line + `"auction: disabled"`; + - `match --details` → slot div ID, GAM unit path, formats, providers lines; + - `check --expect-no-slots` → success message; + - `check` failure with missing and unexpected slots → the exact failure lines; + - `explain` → each gate line (including `"auction providers configured"`) and the + EdgeZero legacy-fallback warning text. + + Run the existing tests first and copy the real emitted strings so the + characterization assertions match current behavior exactly before refactoring. + +- [ ] **Step 2: Run tests before refactor** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: characterization tests pass against the current implementation. + +- [ ] **Step 3: Move formatting into `ad_templates::output`** + + Move these helpers out of `config_ad_templates.rs`: + - `write_match_result` + - `write_gate` + - `format_slot` + - `format_format` + - `format_providers` + - `join_set` + - `plural` + + Keep command functions small: parse args, load config, call expected/gate logic, render. + +- [ ] **Step 4: Reuse shared gate helper in `explain`** + + Build `AdStackGateInput` from explain flags and config: + + ```rust + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get, + navigation: !args.non_navigation, + prefetch: args.prefetch, + bot: args.bot, + matched_slots: !expected.slots.is_empty(), + consent_allows_auction: Some(!args.consent_denied), + auction_enabled: loaded.settings.auction.enabled, + }); + ``` + + Render the seven shared gate names from `gate` rather than hand-rolled boolean chains. + + **Preserve the explain-only provider gate.** The current `run_explain` + (`config_ad_templates.rs:270`) renders an eighth gate, + `"auction providers configured"` (`!loaded.settings.auction.providers.is_empty()`, + line 299), and ANDs it into its local `runs_ad_stack` decision (line 302). The + shared `evaluate_ad_stack_gate` helper and runtime `should_run_server_side_ad_stack` + intentionally have no provider-configured gate. Do not fold this into + `AdStackGateInput`. Keep `"auction providers configured"` as an explain-only + supplementary `write_gate(...)` line rendered alongside the shared result, and + keep it in `explain`'s own `runs_ad_stack` decision: + + ```rust + let providers_configured = !loaded.settings.auction.providers.is_empty(); + render_shared_gates(out, &gate)?; + write_gate(out, "auction providers configured", providers_configured)?; + let runs_ad_stack = + gate.expected == RuntimeAdStackExpected::Yes && providers_configured; + ``` + + This keeps `explain` output and behavior identical to the current implementation + (verified by the Step 1 characterization test) while still sharing the seven core + runtime gates with `publisher.rs`. + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: no output regressions except intentional wording updates covered by tests. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/ad_templates/output.rs crates/trusted-server-cli/src/run.rs + git commit -m "Refactor static ad-template commands" + ``` + +## Task 7: Port Generic Audit Namespace And Browser Collector + +**Files:** + +- Modify: `Cargo.toml` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Create: `crates/trusted-server-cli/src/audit/mod.rs` +- Create: `crates/trusted-server-cli/src/audit/page.rs` +- Create: `crates/trusted-server-cli/src/audit/collector.rs` +- Create: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add browser dependencies** + + Add `chromiumoxide` to the root `[workspace.dependencies]` table (inert until a + crate references it via `{ workspace = true }`): + + ```toml + [workspace.dependencies] + # ... existing entries ... + chromiumoxide = "0.9.1" + ``` + + Add the host deps to the CLI crate under its existing + `[target.'cfg(not(target_arch = "wasm32"))'.dependencies]` table — NOT a plain + `[dependencies]` table (workspace default target is wasm32; an unconditional dep + compiles for wasm and breaks the build / leaks host-only crates): + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] + # ... existing clap/url/serde/etc ... + chromiumoxide = { workspace = true } + futures = { workspace = true } + tempfile = { workspace = true } + tokio = { workspace = true } + which = { workspace = true } + ``` + + Verify the root workspace already provides `futures`, `tempfile`, `tokio`, `which` + (it does on this branch); only `chromiumoxide` is a new workspace entry. + +- [ ] **Step 2: Write failing audit parser tests** + + In `run.rs` tests: + + ```rust + #[test] + fn audit_legacy_url_parses_as_page_alias() { + let args = parse(&["ts", "audit", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_page_subcommand_parses() { + let args = parse(&["ts", "audit", "page", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_ad_templates_verify_parses() { + let args = parse(&["ts", "audit", "ad-templates", "verify", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_ad_templates_is_not_legacy_url() { + assert!(Args::try_parse_from(["ts", "audit", "ad-templates"]).is_err()); + } + ``` + +- [ ] **Step 3: Run parser tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli audit_ --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because `Audit` command does not exist. + +- [ ] **Step 4: Implement audit Clap namespace in current `run.rs` shape** + + Do not add stale #800 `args.rs`. Add a `Command::Audit(AuditArgs)` variant to the + existing `Command` enum, plus the full audit arg surface. The parser tests in + Step 2 exercise `ad-templates verify`, so the **entire** command surface (including + the verify args) must be defined here for those tests to compile. Task 9 implements + the verify _behavior_ only — it does not redefine these arg types. + + **Visibility:** `audit::run_audit` lives in `audit/mod.rs` and must name these + types in its signature and match their variants, so every audit arg type and its + fields are `pub(crate)` (not private). `PageAuditArgs` (from `audit/page.rs`, Task 7 + Step 5) and `AuditAdTemplatesVerifyArgs` are likewise `pub(crate)`/`pub`. `run.rs` + imports `AuditArgs` for the `Command::Audit(AuditArgs)` variant; everything else is + read by `audit/mod.rs`. (This mirrors `config_ad_templates::AdTemplatesCommand`, + which is `pub` and consumed by `run.rs`.) + + ```rust + // value parser shared by legacy_url and verify urls; rejects non-HTTP(S) schemes. + pub(crate) fn parse_http_url(raw: &str) -> Result { + let url = url::Url::parse(raw).map_err(|error| format!("invalid URL `{raw}`: {error}"))?; + match url.scheme() { + "http" | "https" => Ok(url), + other => Err(format!("unsupported URL scheme `{other}` (expected http or https)")), + } + } + + #[derive(Debug, clap::Args)] + pub(crate) struct AuditArgs { + #[command(subcommand)] + pub(crate) command: Option, + #[arg(value_parser = parse_http_url, hide = true)] + pub(crate) legacy_url: Option, + } + + #[derive(Debug, Subcommand)] + pub(crate) enum AuditSubcommand { + Page(PageAuditArgs), + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), + } + + #[derive(Debug, Subcommand)] + pub(crate) enum AuditAdTemplatesCommand { + Verify(AuditAdTemplatesVerifyArgs), + } + + // Defined here (not Task 9) so parser tests compile. Task 9 fills in the handler. + #[derive(Debug, clap::Args)] + pub(crate) struct AuditAdTemplatesVerifyArgs { + #[command(flatten)] + pub config: AppConfigArgs, + #[arg(required = true, value_parser = parse_http_url)] + pub urls: Vec, + #[arg(long)] + pub strict: bool, + #[arg(long)] + pub json: bool, + #[arg(long)] + pub scroll: bool, + } + ``` + + Dispatch `Command::Audit(args)` to a single `audit::run_audit(args: AuditArgs)` + entry point (in `audit/mod.rs`) that normalizes the namespace: `legacy_url` (if + present) and `Page` both route to the generic page audit; `AdTemplates(Verify(..))` + routes to the verifier (a stub returning `Ok(())` until Task 9). If Clap cannot make + the optional-subcommand-plus-hidden-positional contract unambiguous, implement a + small `AuditArgs::normalize()` that rejects `legacy_url` values that are not HTTP(S). + Decide arg-type home consistently: keep them in `run.rs` as `pub(crate)` (as shown) + and import into `audit/mod.rs`, or move them next to `run_audit` in `audit/mod.rs` + and import `AuditArgs` into `run.rs` — either works, but do not split them. + +- [ ] **Step 5: Port minimal generic page audit** + + Port useful #800 concepts into `audit/page.rs`, but keep output read-only by default for now: + - parse/validate URL; + - call `AuditCollector::collect_page`; + - print summary with final URL, title, script/resource counts, warnings; + - no draft config generation in this PR unless #800 rebase keeps it explicitly. + +- [ ] **Step 6: Implement collector trait and browser collector base** + + `audit/collector.rs` — define the trait plus its concrete request/response types so + Task 9's `FakeCollector` and the verify orchestration have a contract to assert on: + + ```rust + pub trait AuditCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result; + } + + pub struct BrowserCollectRequest { + pub url: url::Url, + // Pre-navigation init scripts (evaluate-on-new-document). Empty for plain page audit; + // Task 8 passes the ad-template collector script here. + pub init_scripts: Vec, + pub scroll: bool, + } + + pub struct CollectedPage { + pub final_url: url::Url, + pub title: String, + // Generic page-audit signals (counts only; no page HTML/cookies/storage). + pub script_count: usize, + pub resource_count: usize, + pub warnings: Vec, + // Present only when an ad-template init script was injected (Task 8); None for + // plain `ts audit page`. This is how `BrowserAdEvidence` rides on a CollectedPage. + pub ad_evidence: Option, + } + ``` + + `BrowserCollectRequest` carries `init_scripts` + `scroll` so ad-template verification + enables evidence hooks without changing the trait later. + + `audit/browser.rs` should port #800's: + - `which` browser lookup; + - isolated `TempDir` profile; + - current-thread Tokio runtime; + - `Browser::launch`; + - `page.goto`; + - `wait_for_navigation_response`; + - settle loop. + +- [ ] **Step 7: Run compile-focused CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli audit_ --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: parser and non-browser unit tests pass. No test should require installed Chrome yet. + +- [ ] **Step 8: Commit** + + ```bash + git add Cargo.toml crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/audit crates/trusted-server-cli/src/lib.rs crates/trusted-server-cli/src/run.rs + git commit -m "Add audit namespace and browser collector base" + ``` + +## Task 8: Add Browser Ad-Template Evidence Collector + +**Files:** + +- Create: `crates/trusted-server-cli/src/audit/ad_template_collector.js` +- Modify: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/compare.rs` + +- [ ] **Step 1: Write JS collector contract fixture tests** + + Add Rust unit tests that inspect generated init-script text and decode a mocked `window.__tsAdTemplateEvidence` JSON payload. These should not launch Chrome. + + Prefer **behavioral** assertions over brittle substring matching: where possible, + assert by decoding a mocked `window.__tsAdTemplateEvidence` payload into + `BrowserAdEvidence` and checking fields. For the few structural checks that must + inspect the script text, pin **exact** marker substrings (no "or equivalent", so + the pass condition is deterministic) — choose the markers to match the strings the + implementation will actually emit: + - `build_ad_template_init_script` output contains the literal `__TS_CONFIG` injection; + - contains the chosen googletag-hook marker (pick ONE and pin it, e.g. + `Object.defineProperty(window, "googletag"`); + - contains the `cmd.push` wrap marker; + - contains the `defineSlot` record marker; + - contains the `apstag.fetchBids` wrap marker; + - embeds only the configured div prefixes / provider IDs passed via `__TS_CONFIG` + (assert a non-configured prefix is absent). + +- [ ] **Step 2: Run tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_template_collector --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: failure because collector script/builder does not exist. + +- [ ] **Step 3: Implement init script builder** + + In Rust, build script as: + + ```rust + pub fn build_ad_template_init_script(config: &AdTemplateCollectorConfig) -> Result { + let config_json = serde_json::to_string(config) + .map_err(|error| format!("failed to serialize ad-template collector config: {error}"))?; + Ok(format!(";(() => {{ const __TS_CONFIG = {config_json};\n{}\n}})();", include_str!("ad_template_collector.js"))) + } + ``` + + Keep the JS file generic; pass configured prefixes and APS slot IDs through `__TS_CONFIG`. + +- [ ] **Step 4: Implement read-only JS evidence collection** + + In `ad_template_collector.js`, write to `window.__tsAdTemplateEvidence`: + - `dom_ids`: matched IDs from configured prefixes, excluding `-container`; + - `gpt_slots`: record `defineSlot` calls observed **both** directly **and** when + dispatched from the `googletag.cmd` queue (wrap `cmd.push` so queued callbacks are + instrumented without changing their order — spec §7), **plus** a post-settle + `googletag.pubads().getSlots()` scrape. For each scraped slot capture + `getAdUnitPath()`, `getSlotElementId()`, and `getSizes()` so `getSlots()`-only + slots still carry numeric `sizes` for the §5.4 overlap rule. Normalize sizes from + both `defineSlot` input and `getSizes()` output: `[300,250]` → one `(300,250)`; + `[[300,250],[728,90]]` → two pairs; non-numeric (`"fluid"`) dropped from numeric + sizes and surfaced as a `fluid_size_ignored` warning; + - `aps_calls`: `fetchBids` payloads (configured slot IDs + sizes); + - `warnings`: collector warnings only ({code, message}), no page HTML/cookies/storage. + + Always call original page functions with unchanged arguments, and never override + `navigator.webdriver` (spec §7). + +- [ ] **Step 5: Add browser collector extraction** + + After settle and after optional scroll, evaluate: + + ```javascript + ;() => window.__tsAdTemplateEvidence || null + ``` + + Decode into `BrowserAdEvidence`. If decode fails, return a page warning rather than failing navigation. + +- [ ] **Step 6: Add deterministic scroll** + + In `audit/browser.rs`, implement `scroll` by evaluating: + + ```javascript + ;async () => { + const height = Math.max( + document.body.scrollHeight, + document.documentElement.scrollHeight + ) + for (const y of [ + Math.floor(height * 0.33), + Math.floor(height * 0.66), + height, + ]) { + window.scrollTo(0, y) + await new Promise((resolve) => setTimeout(resolve, 250)) + } + window.scrollTo(0, 0) + } + ``` + + Then wait for the same settle quiet period and collect evidence with `phase = "scroll"` where the JS script marks new observations. + +- [ ] **Step 7: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_template_collector --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli audit::browser --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: unit tests pass without launching Chrome. + +- [ ] **Step 8: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/ad_template_collector.js crates/trusted-server-cli/src/audit/browser.rs crates/trusted-server-cli/src/audit/collector.rs crates/trusted-server-cli/src/ad_templates/compare.rs + git commit -m "Collect browser ad-template evidence" + ``` + +## Task 9: Implement `ts audit ad-templates verify` + +**Files:** + +- Create: `crates/trusted-server-cli/src/audit/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` + +- [ ] **Step 1: Write failing orchestration tests with a fake collector** + + Build a fake collector implementing `AuditCollector` and test: + - one confirmed page exits success in default mode; + - strict missing slot returns error; + - `[auction].enabled = false` returns runtime skipped and does not strict-fail missing evidence; + - one page navigation error plus one success sets JSON `ok = false`; + - invalid `ftp://` URL fails before fake collector is called; + - redirect uses final path for expected slots and emits redirect warning. + + Define the test scaffolding explicitly (no dangling helpers): + + ```rust + // Maps each requested URL to a canned outcome so orchestration is tested without Chrome. + struct FakeCollector { + pages: std::collections::HashMap>, + } + + impl FakeCollector { + // Success page: requested -> final_url, carrying the given ad evidence. + fn page(requested: &str, final_url: &str, evidence: BrowserAdEvidence) -> Self { + let mut pages = std::collections::HashMap::new(); + pages.insert( + requested.to_string(), + Ok(CollectedPage { + final_url: url::Url::parse(final_url).expect("valid final url"), + title: String::new(), + script_count: 0, + resource_count: 0, + warnings: Vec::new(), + ad_evidence: Some(evidence), + }), + ); + Self { pages } + } + // Helper to add a failing page for multi-URL tests. + fn with_error(mut self, requested: &str, message: &str) -> Self { + self.pages.insert(requested.to_string(), Err(message.to_string())); + self + } + } + + impl AuditCollector for FakeCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.pages + .get(request.url.as_str()) + .cloned() + .unwrap_or_else(|| Err(format!("no fake page for {}", request.url))) + } + } + + impl BrowserAdEvidence { + // #[cfg(test)] fixture: one confirmed news slot (atf / ad-atf-0 / /123/news/atf, 300x250). + fn confirmed_news_slot() -> Self { + BrowserAdEvidence { + dom_ids: vec![DomEvidence { dom_id: "ad-atf-0".into(), phase: EvidencePhase::InitialLoad }], + gpt_slots: vec![GptSlotEvidence { + gam_unit_path: "/123/news/atf".into(), + div_id: "ad-atf-0".into(), + sizes: vec![(300, 250)], + phase: EvidencePhase::InitialLoad, + }], + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + } + + // Runs the verify orchestration with `--json` over `urls` and returns parsed JSON. + // Loads a #[cfg(test)] effective config whose `/news/*` slot is the atf slot above. + fn run_verify_json(collector: &dyn AuditCollector, urls: impl IntoIterator) -> serde_json::Value { /* impl in test module */ } + + #[test] + fn verify_uses_final_url_for_matching_after_redirect() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + BrowserAdEvidence::confirmed_news_slot(), + ); + let json = run_verify_json(&collector, ["https://www.example.com/"]); + + assert_eq!(json["pages"][0]["path"], "/news/story"); + // Warning order is unspecified; assert presence, not index 0. + let warnings = json["pages"][0]["warnings"].as_array().expect("warnings array"); + assert!( + warnings.iter().any(|w| w["code"] == "redirected"), + "redirect should emit a `redirected` warning" + ); + } + ``` + + `run_verify_json` calls the same `run_verify` entry point used in production but + with the fake collector injected and output captured; define it in the test module + so all six listed cases share it. + +- [ ] **Step 2: Run focused tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli audit::ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because verifier module does not exist. + +- [ ] **Step 3: Wire the verifier handler** + + `AuditAdTemplatesVerifyArgs` already exists from Task 7, Step 4. Replace the Task 7 + stub so `audit::run_audit` routes `AdTemplates(Verify(args))` into a new + `audit::ad_templates::run_verify(args)`. Do not redefine the arg struct. + +- [ ] **Step 4: Implement verification orchestration** + + For each URL: + 1. Collect browser page with ad-template init script and optional scroll. + 2. Parse final URL and normalize final path. + 3. Build expected slots for final path. + 4. Build gate summary using shared core gate helper with `consent_allows_auction = None`. + 5. Add redirect warning (`code = "redirected"`) if requested path differs from final path. + 6. Compare evidence (`compare_page_evidence`) to get a `PageVerificationResult`. + 7. **Assemble the wire `PageJson`** (Task 4 type) from the pieces the comparison + result does not carry: `url` / `final_url` / `requested_path` / `path`, + `gates` (map the gate summary's per-gate states to `GateState`), + `matched_slot_count`, `runtime_ad_stack_expected` (via the `From` impl on + `RuntimeAdStackExpectedJson`), then the `slots` / `extra_evidence` / `warnings` + from the comparison result. `PageVerificationResult` is intentionally URL- and + gate-agnostic; this step is where per-page request context is joined in. + 8. Preserve page-level errors as a `PageJson` with `error: Some(..)` and continue + remaining URLs. + +- [ ] **Step 5: Implement exit behavior** + - Default auditor-assist mode: return `Ok(())` for missing/partial evidence when no page-level collection errors occur. + - `--strict`: return `Err(String)` when any non-skipped page has missing/partial slot. + - Multi-URL page errors: JSON `ok=false`; command returns `Err(String)` after writing JSON/human output. + - Invalid schemes: fail before browser launch and before any output. + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli audit::ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: verifier and pure comparison tests pass. + +- [ ] **Step 7: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/ad_templates.rs crates/trusted-server-cli/src/audit/mod.rs crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/ad_templates/output.rs + git commit -m "Verify ad-template slots from browser evidence" + ``` + +## Task 10: Add Local Browser Fixture Tests + +**Files:** + +- Modify: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/audit/ad_templates.rs` + +- [ ] **Step 1: Add test-only local HTTP fixture helper** + + In `audit::browser` tests, create a `TcpListener` serving static HTML from strings. Keep it test-only and host-target only. + + Fixture pages: + - direct `googletag.defineSlot`; + - `googletag.cmd.push`; + - late `window.googletag = { cmd: [] }`; + - late `window.apstag`; + - lazy slot created after scroll; + - redirect from `/` to `/news/story`; + - navigation returning 500. + +- [ ] **Step 2: Gate tests when Chrome is unavailable** + + Add helper: + + ```rust + fn chrome_available() -> bool { + ["chrome", "chromium", "google-chrome", "google-chrome-stable"] + .iter() + .any(|name| which::which(name).is_ok()) + } + ``` + + Each browser fixture test should early-return when unavailable. Do not use + `println!` / `eprintln!`; keep the skip reason in the helper name or a skipped + assertion message so clippy stays clean. This keeps CI portable unless Chrome is + installed. + +- [ ] **Step 3: Write fixture tests** + + Tests should assert the collector sees evidence, not real ad network behavior: + - direct GPT evidence confirms; + - command-queue GPT evidence confirms; + - APS `fetchBids` evidence removes APS provider warning; + - lazy slot appears only when `--scroll` is set; + - redirect result uses final path; + - failed page produces page-level error while other pages continue. + +- [ ] **Step 4: Run fixture tests locally** + + Run: + + ```bash + cargo test -p trusted-server-cli browser_fixture --target $(rustc -vV | sed -n 's/^host: //p') -- --nocapture + ``` + + Expected: pass when Chrome/Chromium exists; otherwise tests skip with explicit message. + +- [ ] **Step 5: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/browser.rs crates/trusted-server-cli/src/audit/ad_templates.rs + git commit -m "Add browser fixtures for ad-template verification" + ``` + +## Task 11: Update Documentation And Help Snapshots + +**Files:** + +- Modify: `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` if implementation decisions differ. +- Modify: `trusted-server.example.toml` only if command examples need harmless fictional config comments. +- Modify: `CLAUDE.md` only if verification commands or CLI command surface need to be documented. + +- [ ] **Step 1: Run CLI help manually** + + Run: + + ```bash + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- audit --help + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- audit ad-templates verify --help + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- config ad-templates --help + ``` + + Expected: nested audit commands are discoverable; hidden legacy `ts audit ` does not dominate help text. + +- [ ] **Step 2: Update docs if help text or behavior differs from spec** + + Keep examples using `https://www.example.com/` only. Do not mention real publisher sites. + +- [ ] **Step 3: Run docs format** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: Prettier passes. + +- [ ] **Step 4: Commit** + + ```bash + git add docs trusted-server.example.toml CLAUDE.md + git commit -m "Document ad-template CLI verification" + ``` + + If no docs changed, skip the commit. + +## Task 12: Final Verification + +**Files:** + +- Verify all touched files. + +- [ ] **Step 1: Rust format** + + Run: + + ```bash + cargo fmt --all -- --check + ``` + + Expected: pass. + +- [ ] **Step 2: Host CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: pass. Browser fixture tests either pass or explicitly skip when Chrome/Chromium is unavailable. + +- [ ] **Step 3: Workspace tests** + + Run: + + ```bash + cargo test --workspace + ``` + + Expected: pass. + +- [ ] **Step 4: Clippy** + + Run: + + ```bash + cargo clippy --workspace --all-targets --all-features -- -D warnings + ``` + + Expected: pass. + +- [ ] **Step 5: Wasm isolation proof** + + `trusted-server-adapter-fastly` does **not** depend on `trusted-server-cli`, so the + adapter build never compiles the CLI crate and cannot detect a CLI-crate dep leak. + The real proof is building the **CLI crate itself** for the wasm target (its modules + are `#[cfg(not(target_arch = "wasm32"))]`, so a wasm build must succeed with the + host-only deps compiled out). Note the workspace default target is already + `wasm32-wasip1`, so Steps 3–4 (`cargo test/clippy --workspace`) also build the CLI + crate for wasm — but make the isolation check explicit: + + ```bash + # Real CLI isolation proof: CLI crate must build for wasm with host deps excluded. + cargo build --package trusted-server-cli --target wasm32-wasip1 + # Adapter still built to confirm the production artifact is unaffected. + cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 + ``` + + Expected: both pass. If `chromiumoxide`/`tokio`/etc. leaked into a non-target-cfg + dependency table, the first command fails — that is the guard. + +- [ ] **Step 6: Docs format** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: pass. + +- [ ] **Step 7: Inspect final diff** + + Run: + + ```bash + git status --short + git diff --stat origin/server-side-ad-templates-impl...HEAD + git log --oneline origin/server-side-ad-templates-impl..HEAD + ``` + + Expected: only intended CLI/core/doc files changed; no `.env`, operator `trusted-server.toml`, or generated browser artifacts included. + +## Risks And Watch Points + +- `chromiumoxide` must remain a host-only `trusted-server-cli` dependency. Any wasm build failure here means the dependency leaked. +- `ts audit ` compatibility must not swallow `ts audit ad-templates` as a URL. +- Runtime gate extraction (Task 1) only touches `should_run_server_side_ad_stack` + (the navigation gate). `/__ts/page-bids` is **intentionally NOT routed** through + `evaluate_ad_stack_gate` — its gate semantics differ (bot/prefetch skip the auction + but keep slots; no `is_navigation`/`is_get` gate). Its parity is preserved by + leaving it untouched, not by sharing the helper. Do not reroute page-bids. Keep + existing publisher and page-bids tests passing. +- The browser collector must not capture page HTML, cookies, storage, request bodies, or arbitrary DOM. Only collect configured-prefix DOM IDs and ad-related evidence. Never override `navigator.webdriver`. +- `runtime_ad_stack_expected = "unknown"` is normal for live consent state; do not over-model consent unless the collector can prove it. +- Browser fixture tests must not depend on real GPT/APS network calls. +- **Deferred in this implementation:** `/__ts/page-bids` SPA observation (spec §5.2 + "when available"). `PageBidsEvidence` exists as forward scaffolding but is not + collected (Task 8), surfaced in JSON (Task 4), or tested. Revisit if SPA route + verification is prioritized. +- Keep generation (`ts audit ad-templates generate`) out of this PR. diff --git a/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md b/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md new file mode 100644 index 000000000..8546ebf6e --- /dev/null +++ b/docs/superpowers/plans/2026-07-06-cache-control-header-implementation-plan.md @@ -0,0 +1,456 @@ +# Cache-Control Header Strategy Implementation Plan + +**Date:** 2026-07-06 +**Status:** Initial cache-header slice implemented in the current branch +**Spec:** `docs/superpowers/specs/2026-07-06-cache-control-header-design.md` + +## Scope + +Implement the **initial cache-header slice** from the current spec. The latest +spec resolves the initial-slice open questions and defers the larger dynamic +caching, template caching, streaming, and compression-offload work. + +Initial slice goals: + +1. Make TS-owned, hash-versioned TSJS responses cache correctly. +2. Make neutralized publisher Prebid compatibility responses safe to cache. +3. Add a structured, runtime-portable cache-policy model. +4. Add a configurable static/rehosted asset cache-rule engine so framework + assumptions are operator-controlled, not hard-coded. +5. Keep arbitrary publisher-origin assets origin-controlled unless an enabled + rule proves they are immutable-safe. + +Deferred follow-up features are listed separately below and should not be folded +into the initial cache-header PRs. + +## Decisions locked for the initial slice + +- SSAT-assembled HTML remains `Cache-Control: private, max-age=0` and strips + runtime edge-cache headers (`Surrogate-Control`, `Fastly-Surrogate-Control`, + `CDN-Cache-Control`, and `Cloudflare-CDN-Cache-Control`) whenever the ad stack + can inject per-user slot/bid state. +- TSJS keeps the current `/static/tsjs=...js?v=` canonical URL shape. + Matching hash/version requests receive immutable cache headers; missing or + mismatched hash/version requests keep short TTLs rather than redirecting. +- Runtime cache-key configuration must preserve the `v` query parameter for + `/static/tsjs=`. Fastly and Cloudflare include query strings in default cache + keys, but project-specific query normalization must not drop `v`. +- Framework-specific immutable paths, including Next.js `/_next/static/*`, must + be represented as configurable cache-rule presets. Do not add adapter- or + proxy-level hard-coded framework path checks. +- Operators decide which framework presets and publisher allowlists are enabled. + Arbitrary publisher CSS/JS/images remain origin-controlled unless an enabled + cache rule proves they are immutable-safe. +- TS-owned Prebid delivery is covered by deferred TSJS module URLs. Publisher + Prebid script URLs neutralized by TS are compatibility shims at stable URLs and + must use `no-store` or a very short TTL, not a year-long immutable policy. +- Fastly rehosted assets are TS-owned copies once TS rewrites/hosts them. A + matching rehost rule is authoritative over third-party origin cache defaults. Use + immutable only for TS-fingerprinted rehosted URLs, and preserve any later + TS/operator `private` or `no-store` decision as the final veto. +- Fastly and Cloudflare are the MVP runtime targets. This slice emits their + runtime-specific directives; actual Fastly storage integration and cache-key + verification are tracked in [#908](https://github.com/IABTechLab/trusted-server/issues/908). + Akamai mapping is deferred until Akamai is on the roadmap. +- Dynamic HTML/RSC/API caching, dynamic `Vary`/cache-key normalization, + origin-template caching, transformed-template caching, true publisher-origin + streaming, parser-context bid splice, EdgeZero streaming parity, and SSAT HTML + compression offload are deferred follow-up features. +- All personalized/cookie-bearing response hardening in `response_privacy.rs` and + adapter middleware stays in place and runs after any new policy application. + +## Original baseline before this implementation + +| Area | Current file(s) | Baseline | +| ---------------------------- | --------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| TSJS URL injection | `crates/trusted-server-core/src/tsjs.rs` | Injects `/static/tsjs=...js?v=`; current branch is moving hash work out of the hot path. | +| TSJS serving | `publisher.rs`, `http_util.rs` | Historically served through `serve_static_with_etag` with 5-minute browser/edge TTLs. | +| Cache policy primitives | `cache_policy.rs` | Current branch adds typed policy rendering; still needs final alignment with no-store and config rules. | +| Neutralized Prebid shim | `crates/trusted-server-core/src/integrations/prebid.rs` | `handle_script_handler` currently returns an empty JS shim with `public, max-age=31536000`; this must be changed. | +| Cache privacy | `publisher.rs`, `response_privacy.rs`, adapter middleware | Ad-stack HTML and cookie-bearing responses are downgraded to private/shared-uncacheable. | +| Rehosted asset cache policy | `proxy.rs` | Policy is effectively origin-controlled or `no-store, private`; no normalized immutable/SWR policy. | +| Dynamic HTML/RSC/API caching | none | Deferred. No initial-slice `Vary` rewriting or Next router-header special casing. | +| Origin-template cache | none | Deferred. No cache API/override/template key/surrogate-key implementation exists. | + +## Definition of done for the initial slice + +- TSJS hash-version-matching requests emit one-year immutable browser cache and + one-year edge cache headers. +- TSJS missing/mismatched hash requests keep short TTL behavior. +- TSJS injected hash generation no longer concatenates and hashes the full + bundle on every page view. +- Neutralized publisher Prebid shim responses use `no-store` or a very short TTL. +- Cache policy is represented as structured data and can emit Fastly + `Surrogate-Control`, generic `CDN-Cache-Control`, Cloudflare-specific + `Cloudflare-CDN-Cache-Control`, and `s-maxage` fallback headers. +- Cache policy can represent `no-store`/uncacheable responses as well as public + and private TTL policies. +- TS config expresses static/rehosted cache policy through structured rules with + match criteria, policy fields, and `enabled` flags. +- Built-in framework presets, including Next.js `/_next/static/*`, are + implemented through the shared rule engine and can be disabled/overridden. +- Arbitrary publisher-origin assets remain origin-controlled unless matched by an + enabled preset or publisher allowlist. +- Fastly TS-owned rehosted assets have explicit normalized policies instead of + blindly passing through third-party defaults. +- MVP adapters emit the correct edge-cache header from shared policy: Fastly + `Surrogate-Control`, Cloudflare `CDN-Cache-Control` / + `Cloudflare-CDN-Cache-Control`, or portable `s-maxage` fallback. Header + emission is complete; runtime storage and cache-key verification remain in + #908. +- Deferred features are documented as deferred and are not accidentally + implemented as hard-coded Next.js/dynamic-cache behavior. +- Tests and target-matched checks pass for touched crates/adapters. + +## Proposed PR sequence + +### PR 1 — Structured cache policy primitives + +Status: implemented in the current branch. + +#### Code changes + +- Keep/add a core module such as `crates/trusted-server-core/src/cache_policy.rs`. +- Define structured policy types: + - `CacheVisibility::{Public, Private}` + - `CachePolicy { visibility, browser_ttl, edge_ttl, stale_while_revalidate, +stale_if_error, immutable }` + - a `no-store` / uncacheable representation, either as a policy mode or a + dedicated helper, so neutralized shims and error responses do not need + ad-hoc strings; + - `EdgeCacheHeader::{SurrogateControl, CdnCacheControl, +CloudflareCdnCacheControl, SMaxageFallback, None}`. +- Add helpers that render policy into headers: + - browser `Cache-Control` + - Fastly `Surrogate-Control` + - generic `CDN-Cache-Control` + - Cloudflare-specific `Cloudflare-CDN-Cache-Control` + - portable `s-maxage` fallback. +- Keep helpers side-effect-limited: they should only mutate cache headers they + own and should not bypass `response_privacy` hardening. When applying private + or no-store policies, remove any existing edge-cache headers owned by the + helper so stale `Surrogate-Control`/CDN cache headers cannot survive. +- Add default policy constructors/constants for: + - immutable static; + - short TSJS fallback; + - neutralized Prebid shim (`no-store` or very short TTL); + - uncacheable private. + +#### Tests + +- Unit-test exact header rendering for immutable, short edge/browser split, + private, no-store, SWR/SIE, generic CDN, Cloudflare-specific CDN, and fallback + `s-maxage` policies. +- Test that `immutable` is omitted when browser TTL is absent or zero. +- Test that edge-header output is disabled for private/no-store responses, and + that applying private/no-store removes any pre-existing edge-cache header the + helper owns. + +### PR 2 — TSJS immutable hash-version serving + +Status: implemented in the current branch with runtime-specific edge-header +selection. + +#### Code changes + +- Extend `crates/trusted-server-js/build.rs` generated metadata with per-module + SHA-256 hashes. +- Update `trusted-server-js/src/bundle.rs`: + - `single_module_hash(id)` returns generated hash instead of hashing content; + - `concatenated_hash(ids)` hashes incrementally without concatenating a full + `String`, or caches the result per normalized module-id set; + - `concatenate_modules(ids)` can remain for serving the response body. +- Update `handle_tsjs_dynamic` in `publisher.rs`: + - parse `?v=` from the request URI; + - compare it with the canonical hash for the requested bundle; + - if it matches, apply immutable static policy plus `Vary: Accept-Encoding`, + ETag, and `X-Compress-Hint: on`; + - if missing/mismatched, keep short TTL policy plus ETag and + `X-Compress-Hint: on`. +- Keep the current canonical path shape (`/static/tsjs=...js?v=`). +- Document/verify that runtime cache-key configuration preserves the `v` query + parameter for `/static/tsjs=`. + +#### Tests + +- `tsjs_script_src` and deferred script tests still produce `?v=`. +- Matching `?v=` returns: + - `Cache-Control: public, max-age=31536000, immutable` + - runtime edge header via policy helper; + - `Vary: Accept-Encoding`; + - ETag. +- Missing/mismatched `?v=` returns short TTL and no `immutable`. +- Deferred disabled module still 404s. +- Hash helpers do not allocate the concatenated body just to hash it. + +### PR 3 — Neutralized publisher Prebid shim cache safety + +Fix the stable publisher Prebid compatibility route separately from TS-owned +Prebid delivery. + +#### Code changes + +- Update `PrebidIntegration::handle_script_handler` in + `crates/trusted-server-core/src/integrations/prebid.rs`. +- Replace the current year-long `public, max-age=31536000` response with either: + - `Cache-Control: no-store`, preferred for compatibility when integration + enablement/config can change; or + - a very short TTL if no-store is too conservative. +- Ensure no `Surrogate-Control`/CDN edge header is emitted for the neutralized + stable URL. +- Keep TS-owned Prebid bundle delivery on the deferred TSJS module path, where + matching `?v=` remains immutable. + +#### Tests + +- Neutralized Prebid script handler returns the empty compatibility script with + `no-store` or the chosen short TTL. +- Neutralized Prebid shim does not emit immutable or year-long cache headers. +- TSJS deferred Prebid still receives immutable headers when `?v=` matches. + +### PR 4 — Configurable static asset cache-rule engine + +Introduce operator-configurable static asset rules before applying immutable +upgrades to publisher-origin assets. + +#### Code changes + +- Add cache-rule settings rather than hard-coded path checks. Suggested shape: + - `CacheAssetRule { id, enabled, matcher, policy }` + - `CacheAssetMatcher::{PathPrefix, Glob, Regex, Extension, Preset}` + - `CacheAssetPreset::NextJsStatic` expands to `/_next/static/*` when enabled. +- Add cache settings under `Settings` (and `trusted-server.example.toml`) with + `#[serde(deny_unknown_fields)]` validation consistent with the rest of the + config model. +- Add a shared rule evaluator with deterministic precedence. Prefer an ordered + rule list where the first enabled match wins; reject duplicate rule IDs and + invalid matcher combinations during settings validation. +- Ship framework presets as data/config defaults or documented examples, not as + special cases in proxy/adapters. +- The Next.js preset may be present in example config, but operators must be able + to disable/override it. Do not silently apply it through a hard-coded branch. +- Support publisher-defined allowlist rules for other frameworks or + publisher-specific fingerprinted paths. +- Apply immutable policy only when an enabled rule/preset says the URL is + content-addressed, or for TS-owned validated hash URLs such as TSJS. + +#### Tests + +- With the Next.js preset enabled, `/_next/static/*` gets immutable policy. +- With the Next.js preset disabled, the same `/_next/static/*` remains + origin-controlled. +- Publisher-defined allowlist rule can mark a non-Next fingerprinted path + immutable. +- Non-matching publisher asset remains origin-controlled. +- Rule precedence is deterministic. +- Invalid regex/glob/config fails validation clearly. + +### PR 5 — MVP runtime edge-header mapping and docs + +Make the shared policy output explicit per runtime before wiring the rule engine +into more routes. This prevents new code from copying the current core +Fastly-specific `Surrogate-Control` behavior. This phase covers directive +rendering only; runtime storage is tracked in #908. + +#### Code changes + +- Stop requiring core helpers such as `handle_tsjs_dynamic` or + `serve_static_with_etag` to hard-code Fastly's `Surrogate-Control`. +- Choose one adapter boundary pattern and use it consistently: + - pass the runtime `EdgeCacheHeader`/policy emitter into core handlers; or + - return cache-policy metadata in response extensions and let adapters render + runtime-specific headers after route handling. +- Fastly adapter emits `Surrogate-Control` for edge TTLs. +- Cloudflare adapter emits `CDN-Cache-Control` or + `Cloudflare-CDN-Cache-Control`, depending on the chosen adapter convention. +- Portable/local fallback can use `s-maxage` inside `Cache-Control` when no + runtime-specific edge header is available. +- Akamai mapping remains absent/deferred; do not add untested Akamai behavior. +- Update `trusted-server.example.toml` and docs with disabled framework preset + examples and operator-owned allowlist examples. + +#### Tests + +- Fastly TSJS/static policy application emits `Surrogate-Control`. +- Cloudflare TSJS/static policy application emits the selected Cloudflare CDN + cache header and does not emit Fastly-only `Surrogate-Control`. +- Fallback policy emits `s-maxage` only for public/shared-cacheable responses. +- Private/no-store responses remove or avoid all edge-cache headers. + +### PR 6 — Apply static/rehosted policies to proxy responses + +Wire the rule engine into the routes that emit publisher-origin or rehosted +assets, using the runtime edge-header mapping from PR 5. + +#### Code changes + +- Extend `AssetProxyCachePolicy` in `proxy.rs` beyond + `OriginControlled`/`NoStorePrivate`, for example: + - `OriginControlled` + - `NoStorePrivate` + - `Normalized(CachePolicy)` from a matched enabled rule. +- Apply normalized policy at the asset handler, then reapply its runtime edge + directive after route finalization only when the finalized response is still + cacheable. Final `private` or `no-store` directives veto reapplication and + remove edge-cache headers. +- Preserve existing no-store/private handling for errors, signed failures, or + responses that set cookies/security headers. A matched TS-owned rehost rule + intentionally replaces the third-party origin's cache defaults before this + final privacy veto. +- Ensure operator `response_headers` cannot weaken protected private/no-store + decisions. +- For TS-owned rehosted copies: + - use immutable only for fingerprinted TS-owned URLs; + - use conservative edge/browser TTLs for stable rehosted URLs; + - keep dynamic/personalized endpoints uncached. + +#### Tests + +- Rehosted/fingerprinted route matched by an enabled rule gets immutable policy. +- Stable rehosted route gets the configured conservative policy, not a borrowed + third-party `no-store` unless configured. +- Rehosted error responses keep `no-store, private`. +- `Set-Cookie` response remains private/no-store and loses surrogate headers. +- Operator response headers cannot re-enable shared caching for protected + responses. + +## Initial config sketch + +Exact names can change during implementation, but keep the shape structured and +operator-controlled. + +```toml +[[cache.asset_rules]] +id = "nextjs-static" +enabled = false # operators may enable for Next.js publishers +preset = "nextjs-static" +visibility = "public" +browser_ttl_seconds = 31536000 +edge_ttl_seconds = 31536000 +immutable = true + +[[cache.asset_rules]] +id = "publisher-fingerprinted-assets-example" +enabled = false +path_globs = [ + "/assets/**/*.js", + "/assets/**/*.css", + "/assets/**/*.png", + "/assets/**/*.jpg", + "/assets/**/*.webp", + "/assets/**/*.avif", +] +fingerprint_style = "vite-base64-url" +visibility = "public" +browser_ttl_seconds = 31536000 +edge_ttl_seconds = 31536000 +immutable = true + +[cache.tsjs.versioned] +visibility = "public" +browser_ttl_seconds = 31536000 +edge_ttl_seconds = 31536000 +immutable = true + +[cache.tsjs.fallback] +visibility = "public" +browser_ttl_seconds = 300 +edge_ttl_seconds = 300 +stale_while_revalidate_seconds = 60 +stale_if_error_seconds = 86400 + +[cache.prebid_neutralized] +mode = "no-store" +``` + +Defaults should preserve current behavior unless a rule is explicitly enabled or +unless the response is TS-owned and hash-validated, such as TSJS. + +## Deferred follow-up backlog + +These remain valuable, but are intentionally outside the initial cache-header +slice. + +| Follow-up | Why deferred | Notes | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| True publisher-origin streaming | Requires platform/body boundary changes and adapter streaming semantics | Includes avoiding full `take_body_bytes()` materialization on Fastly and documenting/implementing non-Fastly streaming parity. | +| Parser-context bid splice | Requires HTML pipeline redesign | Replace raw ` **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a disabled-by-default `gam_attribution_enabled` GPT option that marks every eligible request from a Trusted Server head-emitted publisher document with the fixed page-level GAM value `ts=true`. + +**Architecture:** One parsed `GptConfig` instance controls both delivery paths. The raw head bootstrap is the primary, earliest queue insertion; integration-owned publisher-tag metadata adds `data-ts-gam-attribution="true"` to the synchronous bundle for a `document.currentScript`-gated fallback. Existing slot targeting, Prebid refresh cleanup, creative-opportunity forwarding, and Fastly streaming behavior remain unchanged. + +**Tech Stack:** Rust 1.95, Serde, `validator`, `lol_html`, TypeScript, Vitest/jsdom, Playwright, Google Publisher Tag + +--- + +**Issue:** [#1027](https://github.com/IABTechLab/trusted-server/issues/1027) + +**Design:** `docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md` + +## Fixed contracts + +| Concern | Contract | +| ------------------ | ---------------------------------------------------------------------------------------------------- | +| Marker | Exactly page-level `ts=true`; no configurable name/value, alias, or dual write | +| Default | `[integrations.gpt] gam_attribution_enabled = false` | +| Kill switch | Disables only attribution; GPT proxying, shim, `adInit`, and `ts_initial` remain active | +| Primary path | Raw GPT bootstrap queues targeting before `if (ts.adInit) return;` | +| Fallback | Existing synchronous publisher bundle, authorized only by its own `document.currentScript` attribute | +| Publisher tag | One tag; `data-ts-gam-attribution="true"` only for enabled GPT attribution | +| Streaming meaning | Rewritten head emitted before the request, not complete response success; no new buffering | +| Slot targeting | `ts_initial=1` lifecycle unchanged; page-level `ts` is never added to cleanup arrays | +| Operator targeting | Forwarded verbatim; characterize collisions but add no validator, filter, or interception | +| Analysis | Descriptive GAM delivery attribution, not a causal treatment effect | + +Run every command block from the repository root unless that block begins with +an explicit `cd`. Treat separate command blocks as separate shell sessions. + +## File map + +| File | Responsibility | +| ------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/integrations/gpt.rs` | Parse the option, emit the inline activation flag, and expose publisher-tag metadata | +| `crates/trusted-server-core/src/integrations/registry.rs` | Define the default-empty tag-attribute hook and aggregate enabled integration metadata | +| `crates/trusted-server-core/src/tsjs.rs` | Render an attributed publisher bundle tag without changing generic/creative tag output | +| `crates/trusted-server-core/src/html_processor.rs` | Pass registry-owned attributes to the single synchronous publisher bundle tag | +| `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` | Queue the primary `setConfig({ targeting: { ts: "true" } })` callback | +| `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` | Queue the `document.currentScript`-authorized fallback | +| `crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts` | Execute the raw bootstrap and prove ordering/failure isolation | +| `crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts` | Prove exact executing-tag activation and fail-closed cases | +| `crates/trusted-server-core/src/creative_opportunities.rs` | Characterize verbatim operator `ts` targeting; production code remains unchanged | +| `crates/trusted-server-core/src/publisher.rs` | Characterize wire forwarding and marked-head-before-EOF streaming | +| `crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts` | Freeze GPT slot cleanup behavior | +| `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` | Characterize Prebid-produced slot-level collisions and cleanup behavior | +| `crates/trusted-server-cli/tests/config_env_overlay.rs` | Prove the typed CLI environment override updates an existing TOML leaf | +| `trusted-server.example.toml` | Publish the disabled default | +| `crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml` | Keep the browser fixture explicitly default-off | +| `crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts` | Smoke-test absence of the activation attribute in the default-off deployment | +| `docs/guide/integrations/gpt.md` | Document configuration, semantics, audit, reporting, and rollback prerequisites | + +## Task 1: Add the GPT attribution option and integration-owned metadata + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/gpt.rs:64-94` +- Modify: `crates/trusted-server-core/src/integrations/gpt.rs:467-510` +- Modify: `crates/trusted-server-core/src/integrations/gpt.rs:549-559` +- Modify: `crates/trusted-server-core/src/integrations/registry.rs:572-579` +- Test: `crates/trusted-server-core/src/integrations/gpt.rs` + +- [ ] **Step 1: Write failing configuration and head-injector tests.** + + Add tests that deserialize omitted, explicit-false, and explicit-true values, + then exercise both activation outputs. Use behavior-oriented assertions like: + + ```rust + #[test] + fn gam_attribution_defaults_to_disabled() { + let config: GptConfig = + serde_json::from_value(serde_json::json!({})).expect("should parse defaults"); + assert!(!config.gam_attribution_enabled); + } + + #[test] + fn gam_attribution_true_adds_both_activation_signals_without_a_new_insert() { + let integration = GptIntegration::new(GptConfig { + gam_attribution_enabled: true, + ..test_config() + }); + let document_state = IntegrationDocumentState::default(); + let context = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + document_state: &document_state, + }; + let inserts = integration.head_inserts(&context); + + assert_eq!(inserts.len(), 2); + assert!(inserts[0].contains("window.__tsjs_gam_attribution_enabled=true;")); + assert_eq!( + integration.tsjs_script_tag_attributes(), + vec![("data-ts-gam-attribution", "true")] + ); + } + ``` + + Retain the current exact-string assertion for the false first insert. Add + `gam_attribution_enabled: false` to `test_config()` and any other full + `GptConfig` literals. + +- [ ] **Step 2: Run the focused tests and confirm RED.** + + Run: + + ```bash + cargo test-fastly gam_attribution + ``` + + Expected: compilation fails because `GptConfig::gam_attribution_enabled` and + `IntegrationHeadInjector::tsjs_script_tag_attributes` do not exist. + +- [ ] **Step 3: Add the default-empty trait hook and parsed field.** + + Add an object-safe default method beside `head_inserts`: + + ```rust + pub trait IntegrationHeadInjector: Send + Sync { + fn integration_id(&self) -> &'static str; + fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec; + + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + Vec::new() + } + } + ``` + + Add the flat field to `GptConfig`: + + ```rust + /// Enable page-level `ts=true` delivery attribution in GAM. + #[serde(default)] + pub gam_attribution_enabled: bool, + ``` + + Do not add a configurable key or value. + +- [ ] **Step 4: Emit the true-only inline flag without changing false bytes.** + + Build the first insert with an empty-or-fixed fragment: + + ```rust + let gam_attribution_flag = self + .config + .gam_attribution_enabled + .then_some("window.__tsjs_gam_attribution_enabled=true;") + .unwrap_or_default(); + + let mut scripts = vec![ + format!( + "" + ), + format!("", GPT_BOOTSTRAP_JS), + ]; + ``` + + Verify the false string remains exactly: + + ```text + + ``` + +- [ ] **Step 5: Override the metadata hook from the same `GptConfig`.** + + ```rust + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + if self.config.gam_attribution_enabled { + vec![("data-ts-gam-attribution", "true")] + } else { + Vec::new() + } + } + ``` + + Do not store this state in `HtmlProcessorConfig` or + `IntegrationDocumentState`. + +- [ ] **Step 6: Run focused and neighboring GPT tests.** + + ```bash + cargo test-fastly gam_attribution + cargo test-fastly head_injector + ``` + + Expected: PASS; false preserves two current inserts, true adds the flag and + metadata while still emitting two inserts when `slim_prebid_url` is absent. + +- [ ] **Step 7: Commit.** + + ```bash + git add crates/trusted-server-core/src/integrations/gpt.rs crates/trusted-server-core/src/integrations/registry.rs + git commit -m "Add GPT GAM attribution option" + ``` + +## Task 2: Put activation metadata on only the publisher bundle tag + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/registry.rs:1043-1054` +- Modify: `crates/trusted-server-core/src/tsjs.rs:11-39` +- Modify: `crates/trusted-server-core/src/html_processor.rs:324-360` +- Test: `crates/trusted-server-core/src/integrations/registry.rs` +- Test: `crates/trusted-server-core/src/tsjs.rs:161-187` +- Test: `crates/trusted-server-core/src/html_processor.rs:768-820` +- Test: `crates/trusted-server-core/src/html_processor.rs:1637-1671` + +- [ ] **Step 1: Write failing registry and tag-rendering tests.** + + Add a test head injector whose metadata method returns the attribution pair. + Assert registry aggregation is deterministic and preserves the default-empty + behavior of injectors that implement only `head_inserts`. + + Add exact tag tests: + + ```rust + #[test] + fn publisher_script_tag_renders_static_attributes() { + let ids = ["gpt"]; + let src = tsjs_script_src(&ids); + + assert_eq!( + tsjs_script_tag_with_attributes( + &ids, + &[("data-ts-gam-attribution", "true")] + ), + format!( + "" + ) + ); + assert_eq!( + tsjs_script_tag(&ids), + format!("") + ); + } + ``` + + The final string must contain no formatting whitespace introduced only by the + multiline example. + +- [ ] **Step 2: Write a failing HTML matrix test.** + + Process `` with: + 1. a real enabled GPT registry with attribution true; + 2. enabled GPT with attribution false; and + 3. no GPT integration. + + Assert exactly one `#trustedserver-js` tag in every case, the attribute only + in case 1, and integration head inserts remain before the external bundle. + Also retain the generic `tsjs_unified_script_tag()` exact-output test so + creative/all-modules callers stay unmarked. + +- [ ] **Step 3: Run the tests and confirm RED.** + + ```bash + cargo test-fastly tsjs_script_tag + cargo test-fastly integration_head_injector + ``` + + Expected: FAIL because the registry aggregator and attributed publisher + helper are not implemented. + +- [ ] **Step 4: Aggregate integration-owned static attributes.** + + Add beside `IntegrationRegistry::head_inserts`: + + ```rust + #[must_use] + pub fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + self.inner + .head_injectors + .iter() + .flat_map(|injector| injector.tsjs_script_tag_attributes()) + .collect() + } + ``` + + Keep the hook default-empty so existing integration injectors and test doubles + compile without changes. + +- [ ] **Step 5: Add the publisher-only tag helper.** + + Render only trusted, compile-time static attribute pairs: + + ```rust + #[must_use] + pub fn tsjs_script_tag_with_attributes( + module_ids: &[&str], + attributes: &[(&'static str, &'static str)], + ) -> String { + let attributes = attributes + .iter() + .map(|(name, value)| format!(" {name}=\"{value}\"")) + .collect::(); + format!( + "", + tsjs_script_src(module_ids) + ) + } + ``` + + Have `tsjs_script_tag(module_ids)` retain its exact output, either directly or + by delegating with an empty slice. Do not change + `tsjs_unified_script_tag()` or either creative call site. + +- [ ] **Step 6: Wire only the publisher HTML path.** + + Replace the single `html_processor.rs` call with: + + ```rust + let immediate_ids = integrations.js_module_ids_immediate(); + let script_attributes = integrations.tsjs_script_tag_attributes(); + snippet.push_str(&tsjs::tsjs_script_tag_with_attributes( + &immediate_ids, + &script_attributes, + )); + ``` + + Preserve source order: ad slots, integration head inserts, diagnostics + bootstrap, one synchronous bundle, diagnostics module, deferred bundles. + +- [ ] **Step 7: Run focused tests.** + + ```bash + cargo test-fastly tsjs_script_tag + cargo test-fastly integration_head_injector + cargo test-fastly golden_script_tag + ``` + + Expected: PASS; false/non-GPT/generic output is unmarked and true output has + one attributed publisher tag. + +- [ ] **Step 8: Commit.** + + ```bash + git add crates/trusted-server-core/src/integrations/registry.rs crates/trusted-server-core/src/tsjs.rs crates/trusted-server-core/src/html_processor.rs + git commit -m "Authorize GAM attribution bundle" + ``` + +## Task 3: Queue the primary marker before the bootstrap guard + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/gpt_bootstrap.js:17-45` +- Modify: `crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts:8-220` +- Test: `crates/trusted-server-core/src/integrations/gpt.rs:1131-1454` + +- [ ] **Step 1: Extend the raw-source test harness.** + + Add the optional page flag and `setConfig` surface: + + ```typescript + interface MockGoogleTag { + cmd: MockCommandQueue + setConfig?: (config: Record) => void + // retain the existing members + } + + type TestWindow = Omit & { + googletag?: MockGoogleTag + tsjs?: Partial + __tsjs_gam_attribution_enabled?: boolean + } + + function makeGoogleTag( + overrides: Partial = {} + ): MockGoogleTag { + return { + cmd: [], + defineSlot: vi.fn(), + pubads: vi.fn(() => ({})), + enableServices: vi.fn(), + display: vi.fn(), + ...overrides, + } + } + ``` + + Delete the flag in both `beforeEach` and `afterEach`. + +- [ ] **Step 2: Write failing behavioral tests.** + + Cover all of these independently: + - default/false plus a preinstalled `ts.adInit` returns without creating + `window.googletag`; + - true queues the exact string-valued targeting callback before a publisher + callback appended after `runBootstrap()`; + - true plus preinstalled `ts.adInit` still queues and applies targeting but + does not replace `adInit` or install the fallback scheduler; + - missing `setConfig` is a no-op and the initial-load detector and `adInit` + still install; + - throwing `setConfig` is caught inside the marker callback, and a later + publisher callback still executes; + - the wrapped `disableInitialLoad` path still records + `ts.gptInitialLoadDisabled`. + + Use a real array queue, append a publisher spy after bootstrap execution, and + drain a snapshot in order: + + ```typescript + const queue: Array<() => void> = [] + const setConfig = vi.fn() + ;(window as TestWindow).googletag = makeGoogleTag({ cmd: queue, setConfig }) + ;(window as TestWindow).__tsjs_gam_attribution_enabled = true + + runBootstrap() + const publisherCommand = vi.fn() + queue.push(publisherCommand) + ;[...queue].forEach((command) => command()) + + expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }) + expect(setConfig.mock.invocationCallOrder[0]).toBeLessThan( + publisherCommand.mock.invocationCallOrder[0] + ) + ``` + +- [ ] **Step 3: Run the raw bootstrap tests and confirm RED.** + + ```bash + cd crates/trusted-server-js/lib + npx vitest run test/integrations/gpt/gpt_bootstrap.test.ts + ``` + + Expected: targeting assertions fail because the raw bootstrap returns before + any marker enqueue. + +- [ ] **Step 4: Implement one flag-gated queue initialization before the guard.** + + Preserve the local `ts` namespace and reuse `tag` in the existing detector: + + ```javascript + var ts = (window.tsjs = window.tsjs || {}) + var tag + + if (window.__tsjs_gam_attribution_enabled === true) { + tag = window.googletag = window.googletag || { cmd: [] } + tag.cmd = tag.cmd || [] + tag.cmd.push(function () { + try { + var gpt = window.googletag + if (gpt && typeof gpt.setConfig === 'function') { + // "ts" is the fixed GAM key, not the local window.tsjs alias. + gpt.setConfig({ targeting: { ts: 'true' } }) + } + } catch (_) { + // Attribution must not interrupt the existing bootstrap queue. + } + }) + } + + if (ts.adInit) return + + tag = tag || (window.googletag = window.googletag || { cmd: [] }) + tag.cmd = tag.cmd || [] + tag.cmd.push(function () { + // existing initial-load detector body, unchanged + }) + ``` + + Do not add a global deduplication state machine, network call, beacon, cookie + read, slot-level key, or third head insert. + +- [ ] **Step 5: Add/retain Rust source-order assertions.** + + In `gpt.rs`, assert the embedded bootstrap's attribution enqueue occurs before + `if (ts.adInit) return;` and before the executable + `googletag.display(` and `googletag.pubads().refresh(` tokens. Do not compare + against comment-only `display()`/`refresh()` text. Retain `ts_initial` + assertions and the two-insert count. + +- [ ] **Step 6: Run focused JS and Rust tests.** + + ```bash + cd crates/trusted-server-js/lib + npx vitest run test/integrations/gpt/gpt_bootstrap.test.ts + cd ../../.. + cargo test-fastly head_inserts + ``` + + Expected: PASS; default behavior is unchanged and every failure mode is + isolated from the existing bootstrap. + +- [ ] **Step 7: Commit.** + + ```bash + git add crates/trusted-server-core/src/integrations/gpt_bootstrap.js crates/trusted-server-core/src/integrations/gpt.rs crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts + git commit -m "Queue page-level GAM attribution" + ``` + +## Task 4: Add the exact-executing-tag bundle fallback + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts:259-307` +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts:1878-1898` +- Modify: `crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts:350-421` + +- [ ] **Step 1: Add a test helper that controls `document.currentScript`.** + + In the runtime-gating suite, install a configurable getter before a fresh + dynamic import and restore it afterward: + + ```typescript + let executingScript: HTMLScriptElement | null + + Object.defineProperty(document, 'currentScript', { + configurable: true, + get: () => executingScript, + }) + ``` + + Use actual ` +``` + +No executable ESI tag. One origin fetch for two requests. `private, no-store` on the hit. +Cached template 353 bytes against 467 served, so the cache holds the pre-assembly +template. All three fragment formats behave: script, JSON, and `400` on a typo. `Inline` +unaffected — two fetches for two requests, no C2 activity, no markers. + +### The bug only a running server could find + +With the auction **enabled**, C2 never engaged: two origin fetches, marker unresolved. + +TS stamps its own `private, no-store` when `should_run_ad_stack` is true. The C2 gate ran +after that stamp, read it as the origin's declaration, concluded `OriginNotShareable`, and +refused — **on every page that serves ads**, which is every page that matters. + +The more important half is why no test caught it. The fixture left the auction disabled +and passed `slots: &[]`, so `should_run_ad_stack` was false in every test, the stamp never +fired, and the ordering was unobservable. Every C2 assertion had been made against the one +configuration where C2's hardest condition does not apply. + +Demonstrated both ways: with the old fixture, reintroducing the bug passes all seven +tests; with the corrected fixture it fails six. + +### Pattern across this branch + +Five bugs now share one shape — compiled, passed every existing test, and were wrong: + +1. The head-seam gate silently disabled body-close injection (`d9e05973`). +2. The key held the encoding the origin _chose_, so the cache could never hit (`2a2e6c6a`). +3. A C2 hit served with no `Cache-Control` at all (`0adb578e`). +4. A C2 hit dropped its in-flight auction, billing SSPs for nothing (`b3ac59a6`). +5. The gate read TS's own header as the origin's (`4c557347`). + +Three were found by writing the test the plan asked for. One needed a running server. None +were found by review — including my own, twice over on the same gate. + +The stale-cache test is the same failure in miniature: it passed while never reaching +`is_stale()`, and only mutation testing exposed that. A test that passes for the wrong +reason is worse than no test, because it is counted as coverage. + +## Independent review — two blockers, and two reasons the cache would have measured nothing + +An independent reviewer read `main...HEAD` and, importantly, **demonstrated** findings by +running code rather than inferring them. Four things it found that review-by-reading had +not. + +**A POST was answered from a cached GET.** `handle_publisher_request` is the `*`-method +fallback route, so a publisher path that renders on GET and accepts a form or webhook on +POST reaches it for both. The origin never saw the mutating request; the caller got `200` +and a page. Fixed at key construction, since the key governs lookup and store alike. + +**`Vary: Accept-Encoding` disqualified everything.** The key has a dedicated +`accept_encoding` field, so such an origin is already keyed correctly — but the coverage +check consulted only the operator-configured list and reported a gap. Every compressing +origin sends that header, so **C2 would have stored nothing against any real origin**. +This is worse than a plain bug: the spike would have measured a hit rate near zero and +reported it as a result. + +**Cookies excluded essentially every repeat visitor.** Any cookie disqualified in both +directions, and TS sets its own identity cookie. The population that could ever see a warm +hit was roughly first-ever page views and cookie-less clients. The design notes called +this the "first-nav exception"; it is the common case, not the exception. Now opt-in via +`origin_is_cookie_independent`, with the `Vary: Cookie` drift guard overriding a wrong +assertion. + +**`ClientFill` had no end-to-end coverage.** The reviewer reintroduced a diagnostics leak +scoped to that mode and all 1889 tests passed. Investigation showed that specific mutation +is unreachable — `requires_private_no_store()` is a strict superset of the injection +condition and stamps before the gate reads headers — but only by a coincidence between two +independent conditions. Both the coverage gap and the coincidence are now pinned by tests. + +### What the review says about the review process + +The reviewer's confirmed findings all came from **running** something. Its clean bills — +no leak in the template itself, no `Inline` regression — came with positive evidence: +tracing that integration context types carry no per-reader field at all, and separately +proving the leakage test has teeth by breaking the store/assemble order and watching it +fail. + +Two of my own comments were wrong, and it caught both by checking rather than reading: +one claimed three call sites where there are two, the other described a dispatcher +mechanism that stopped existing when assembly moved to `CompletedRequest`. + +Verified afterwards against a running server with the origin advertising +`Vary: Accept-Encoding`: a cookie-bearing repeat visitor now costs one origin fetch across +two requests, and a POST still reaches the origin. + +## Step B — consumers of TS's own response headers + +Not yet run. + +## Step C — hold and origin fetch timings + +Not yet run. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md new file mode 100644 index 000000000..5b9a455e7 --- /dev/null +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -0,0 +1,967 @@ +# #1009 ESI Validation Spike + +> **HISTORICAL SPIKE — DO NOT IMPLEMENT.** This document records the investigation, +> including executable ESI tags, parser/subrequests, and a client-fill arm that were all +> removed. Every unchecked item below is historical, not remaining work. The accepted +> implementation keeps the public `esi` spelling but uses Fastly C2 plus exact byte-seam +> assembly. See +> [the merge-hardening design](../specs/2026-08-12-1009-esi-merge-hardening-design.md) and +> [implementation plan](./2026-08-12-1009-esi-merge-hardening.md). + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps +> use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Decide #1009 on evidence. Build a shared-template pipeline behind a flag, run +ESI and client-fill against it, and produce a decision record that either adopts ESI, +adopts client-fill, or rejects both — with the Fastly-only maintenance cost priced in. + +**Architecture:** +`origin → lol_html transform → fastly::cache::core → finalize headers → stream assembly`. + +Headers finalize **before** assembly, not after — streaming responses on this adapter +commit headers first and then pipe chunks, so nothing can be set once assembly starts. + +The transform emits **one unconditional marker at the body-close seam**. Not two: the +head seam is not a template hole, because `tsjs.adSlots` presence is request-gated +(Task 3 Step 2). The cached object is a shared template with no per-user bytes and no +request-dependent decisions. Assembly is either the `esi` crate (edge) or a client fetch +(browser), selected per request by the arm allocator so both are measured on one build. + +**Tech Stack:** Rust 2024, `wasm32-wasip1`, `fastly` 0.12.1 (`cache::core`, `http::purge`), +`esi` 0.7, `lol_html`, a real Fastly test service for cache behaviour. + +**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` — +read the 2026-08-10 correction at the top and +[§6.6](../specs/2026-08-08-esi-cacheable-root-validation-design.md#66-the-esi-pipeline-corrected) +before writing any code. + +**Control:** [the Stage 0 plan](./2026-08-08-1009-measurement-and-stage-0.md). Its +instrumentation and its bypass flag are prerequisites — this plan compares against them +and does not duplicate them. + +--- + +## Why this plan exists + +An earlier revision of the spec concluded ESI was structurally impossible. It was wrong: +`fastly::cache::core` provides the cache boundary natively, and purge runs inside Compute. +That correction reopens #1009 as an empirical question, and this plan is how it gets +answered. + +**What is genuinely uncertain**, and what each arm is for: + +1. Does a shared template plus per-request assembly beat today's inline path enough to + matter? +2. Does **edge** assembly (ESI) beat **client** assembly (a fetch) by enough to justify a + Fastly-only rendering path that must be maintained alongside the portable one? +3. Can per-user leakage be excluded across cold MISS, warm HIT, stale revalidation, + transform failure, and fragment failure? + +Question 3 is a gate, not a metric. A win on 1 and 2 with a failure on 3 is a rejection. + +## Three caches, never conflated + +The original error came from treating these as one thing. Every task below names which it +means. + +| # | Cache | Contents | Status | +| --- | --------------------------------- | ----------------------------- | ----------------------------------- | +| C1 | Origin read-through | raw origin bytes | Exists. Stage 0 turns it back on. | +| C2 | Shared transformed template | post-`lol_html`, pre-assembly | **New.** What this plan builds. | +| C3 | Assembled-response delivery cache | final per-user output | **Must never exist.** Not proposed. | + +If a task appears to require C3, stop — that is the leakage failure mode, not a design +option. + +## Arms + +Five, but only four are treatable as equivalent. + +| Arm | Root | Bids | Notes | +| ------- | ----------------------- | ---------------- | ---------------------------------------------------------- | +| **A0** | inline, C1 bypassed | inline `` | Today. The baseline. | +| **A1** | inline, C1 on | inline `` | Stage 0. Isolates the bypass from the template change. | +| **A2** | shared template from C2 | client fetch | Portable. Works on all four adapters. | +| **A3** | shared template from C2 | ESI at the edge | Fastly-only. The thing #1009 proposed. | +| **REF** | origin direct, TS off | publisher's own | **Reference, not an arm.** Different work, not comparable. | + +A0→A1 measures the bypass. A1→A2 measures the template split. A2→A3 measures edge versus +client assembly — **that difference is the entire case for ESI**, and it is the number +this plan exists to produce. + +**Do not compare A2 and A3 on root TTFB.** They serve the same C2 template, so their root +timings should be near-identical by construction; a null result there proves nothing. +ESI's claimed advantage is that bids arrive without a client round-trip, so measure: +**bids-ready time**, **`adInit` fire time**, and **first TS-attributed creative paint**. +Root TTFB stays as a guard that the template path did not regress, not as the comparison. + +REF is included because #1009 anchors on it, and excluded from pass/fail because TS-off +does no auction and no injection. Comparing against it measures the feature's existence, +not its implementation. + +--- + +## Task order and dependencies + +``` +Task 1 (esi compiles) ── DONE, PASS ──┐ + ├──> Task 3 (C2 cache) ─┬──> Task 4 (A2 client-fill) +Stage 0 plan (flag + instrumentation) ┘ ├──> Task 5 (A3 ESI) + └──> Task 6 (safety gates) + │ + Task 2 (real service) ─────────────────────────────┴──> Task 7 (decision) +``` + +**Task 2 is not a blocker on Tasks 3–6.** Everything those tasks need is exercisable under +Viceroy 0.17 — verified, see Task 2. The real service is required only for the +measurements Task 7 decides on, so provision it once there is something worth measuring. + +Task 6 runs against every arm, not once at the end. + +--- + +## Task 1: Confirm `esi` 0.7 builds on this toolchain + +Cheapest possible falsification. Do this before anything else. + +**Files:** `crates/trusted-server-adapter-fastly/Cargo.toml` + +- [ ] **Step 1: Add the dependency** + +```bash +cargo add esi@0.7 --package trusted-server-adapter-fastly +``` + +It belongs in the **Fastly adapter**, never in `trusted-server-core` — the crate is +hard-bound to `fastly::{Request, Response, Backend}` and core must stay portable. + +- [ ] **Step 2: Check it compiles for the real target** + +```bash +cargo check-fastly +``` + +Expected: clean. The crate declares edition 2021 with no `rust-version`, and pulls recent +`rand` and `nom`, so this is a genuine question on Rust 1.95.0 / `wasm32-wasip1`. + +- [ ] **Step 3: Check no shared dependency was forced to move** + +```bash +git diff --stat Cargo.lock +cargo check --manifest-path crates/trusted-server-integration-tests/Cargo.toml --tests \ + --target "$(rustc -vV | sed -n 's/^host: //p')" +``` + +**Correction, verified 2026-08-10:** an earlier revision of this step warned about a +desync between the root `Cargo.lock` and `crates/trusted-server-integration-tests/Cargo.lock`. +**That second lockfile does not exist** — the crate is a workspace member (root +`Cargo.toml:10`) and shares the root lockfile. The hazard cannot arise in that form. + +What does matter is whether adding `esi` forces an **existing** shared dependency to a new +version, since `regex`, `bytes`, and `log` are used across the workspace. Adding a new +major that coexists is harmless; moving an existing one is not. If one moves, fix with a +targeted `cargo update -p --precise ` — **never a full update**. + +**Already run and recorded** in [the findings](./2026-08-08-1009-measurement-findings.md): +no existing shared dependency moved. + +- [ ] **Step 4: Record and commit, or stop** + +**Task 1 is complete — verdict PASS, recorded 2026-08-10.** `esi` 0.7.1 compiles clean on +Rust 1.95.0 / `wasm32-wasip1`, all six clippy targets pass, and no existing shared +dependency moved. See [the findings](./2026-08-08-1009-measurement-findings.md). + +Had Step 2 failed, this plan would have stopped here with #1009 answered "not on this +toolchain." It did not. + +```bash +git add crates/trusted-server-adapter-fastly/Cargo.toml Cargo.lock +git commit -m "Add the esi crate to the Fastly adapter for the #1009 validation spike" +``` + +--- + +## Task 2: Local validation first, real service only for what needs it + +**Verified 2026-08-10 under Viceroy 0.17: the entire Core Cache surface this spike uses +works locally.** A probe exercised `cache::core::insert`, `lookup`, `finish`, `to_stream`, +and — the shape Task 3 Step 4 actually specifies — `Transaction::lookup`, +`must_insert_or_update`, `insert(...).surrogate_keys(...).execute_and_stream_back()`, and +hit-after-insert semantics. All passed. Recorded in +[the findings](./2026-08-08-1009-measurement-findings.md). + +That reorders this plan. An earlier revision made provisioning a Fastly service Task 2 and +a blocker on everything after it. It is not a blocker: **almost all of the correctness and +safety work is local**, and only the numbers and the cache topology need real +infrastructure. + +| Work | Where | +| ------------------------------------------------------------ | ------------ | +| C2 insert / lookup / transaction logic (Task 3) | **Local** | +| The `lol_html` transform and template byte-identity (Task 3) | **Local** | +| ESI assembly — the crate is pure Rust over `BufRead`/`Write` | **Local** | +| DCA off, dispatcher allowlist, injection refusal (Task 5) | **Local** | +| Fragment-failure degradation (Task 5) | **Local** | +| Header-finalization ordering, no-C3 assertions (Task 6) | **Local** | +| Cross-user leakage / request-neutrality gates (Task 6) | **Local** | +| Shielding behaviour | Real service | +| POP-level cache tiering (`x-cache`, `hit-state`, `age`) | Real service | +| Request collapsing under genuine concurrency | Real service | +| Stale revalidation timing at the edge | Real service | +| **Every performance number in Task 7's decision rule** | Real service | + +**So: build and prove correctness locally through Tasks 3, 5, and 6 before provisioning +anything.** If the design is wrong or leaks, that surfaces locally for free, and the +service is only needed once there is something worth measuring. + +Two caveats on the local scope. Viceroy is a single instance, so a passing `Transaction` +test proves the API works, **not** that collapsing behaves correctly under load. And local +timings are meaningless for the decision — do not let a fast local run substitute for +Task 7 evidence. + +### When the real service is needed + +- [ ] **Step 1: Provision it — after local correctness passes, not before** + +Separate from production. Confirm and record: whether the publisher backend is +**shielded**, and whether any Delivery service fronts the Compute service. Both change +what the numbers mean. + +```bash +fastly service list +fastly backend list --service-id --version latest +``` + +The shielding answer also settles an open question from the Stage 0 findings: #1009's +off-TS win came from a shield HIT, so whether the test service has one determines whether +its numbers transfer to production at all. + +- [ ] **Step 2: Extend the harness for lineage, not just correlation** + +The existing tester-cookie A/B has no way to join server timings to browser timings. A +root-only request ID is not enough either: under A3 the auction happens in a **fragment +subrequest**, so a root ID never reaches the auction telemetry. + +Propagate a **lineage ID plus the experiment arm** through the whole chain: + +``` +root request → C2 lookup → fragment subrequest → auction telemetry → browser render event +``` + +Generated at TS entry, forwarded into the fragment request, attached to the +`auction_events_raw` row, echoed as `x-ts-request-id`, and exposed to the browser harness +so render events carry it. Every timing log line includes both fields. + +Without this the experiment cannot join hold time, origin time, auction telemetry, browser +TTFB, and render outcome for the same pageview. **That is the difference between an +experiment and a pile of numbers.** + +- [ ] **Step 3: Capture C1 and C2 status separately** + +`x-cache`, `hit-state`, and `age` describe the **HTTP read-through cache (C1)**. They say +nothing about the **transformed-template cache (C2)**, which is a `cache::core` object +with no HTTP semantics. Recording only the former and calling it "cache status" would +attribute C2 hits and misses to the wrong tier. + +Emit both: the C1 headers as-is, plus an explicit `x-ts-c2` field carrying HIT / MISS / +STALE / BYPASS from the transaction outcome. Record the serving POP alongside. A median +that mixes cold-MISS and warm-HIT requests is meaningless, and arms cannot be compared +unless the mix is known — per tier. + +- [ ] **Step 4: Build a request-scoped arm allocator** + +`AssemblyMode` as specified in Task 3 is a **global** setting, but the sample plan below +requires randomized, non-sequential allocation. A global flip gives sequential blocks +instead, which confounds arm with time of day, cache warmth, and traffic mix. + +Allocate per request: hash the lineage ID into buckets, or key off the tester cookie. +The global setting stays as the kill switch and as the way to force a single arm; the +allocator is what the experiment actually uses. Record the assigned arm on every log line +and every telemetry row. + +- [ ] **Step 5: Define the sample plan before collecting anything** + +Write all of this into the findings document **before** the first measurement, and treat +it as fixed: + +| Element | What to state | +| -------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Allocation | Requests per arm per route, and how arms are assigned | +| Randomization | Randomized or blocked by route and cache state — not sequential runs | +| Pilot variance | A small pilot to estimate variance, before sizing the real run | +| MDE and power | The smallest difference worth detecting, and the N that detects it | +| CI method | Which interval, computed how | +| Warmup and carryover | How cold MISS is forced, how warm HIT is confirmed, and how one arm's cache state is prevented from contaminating the next | + +Rationale: this whole effort exists because #1009 drew a causal conclusion from N=4 that +did not survive contact with the code. Repeating that with more arms and no power +calculation would be worse, not better — it would look rigorous while being equally +unfalsifiable. + +--- + +## Task 3: Build C2 — the shared transformed-template cache + +The core of the spike. Behind a flag, default off. + +**Files:** + +- `crates/trusted-server-core/src/publisher.rs` — emit **one** unconditional marker at the body-close seam (see Step 2; the head seam is not a template hole) +- `crates/trusted-server-core/src/settings.rs` — the mode flag +- `crates/trusted-server-adapter-fastly/src/` — the `cache::core` read/write + +- [ ] **Step 1: Add the assembly-mode setting** + +```rust +/// How per-user ad state reaches the page. +/// +/// `Inline` is today's behaviour: bids injected before ``, root uncacheable. +/// `ClientFill` and `Esi` both serve a shared template from the transformed-template +/// cache and fill the holes afterwards. Spike-only — remove with the spike. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AssemblyMode { + #[default] + Inline, + ClientFill, + Esi, +} +``` + +Default `Inline` so the flag is a no-op until set. Note the hazards the Stage 0 plan +already documents: `Settings` carries `#[serde(deny_unknown_fields)]`, `ts config push` is +typed, and `Publisher` has a hand-written `Default` plus eight exhaustive test literals +and a live doctest. + +- [ ] **Step 2: Make the template strictly request-neutral** + +**The obvious design is wrong and would leak.** An earlier draft kept `tsjs.adSlots` in +the shared template on the grounds that it is per-URL. Its _content_ is per-URL; its +_presence_ is not. It is gated on `should_run_ad_stack` (`publisher.rs:2920-2927`), which +is `is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the first request to fill C2 would freeze **its own** consent decision, bot +classification, prefetch status, and kill-switch state into an object every later visitor +reads. A consent-denied first fill serves a no-ads template to consenting users; a +consenting first fill serves ad markup to a user who refused. + +**Rule: the template contains an unconditional inert placeholder and nothing else.** + +| Element | Where it lives | +| ------------------------- | -------------------------------------------------- | +| tsjs bundle script tag | Template — content-hashed, genuinely per-URL | +| URL rewrites | Template — per-host, in the cache key | +| `tsjs.adSlots` | **Fragment** — its presence is request-dependent | +| `tsjs.bids` | **Fragment** | +| GPT diagnostics bootstrap | **Fragment** — gated on a per-request cookie/query | + +Emit **one** unconditional marker at the body-close seam, identical on every request that +reaches the transform. Under `Esi` it is an executable ESI include tag; under +`ClientFill` it is nothing at all, with the client fetching unprompted. + +- [ ] **Step 3: Bypass C2 for anything that must not be shared** + +`cache::core` is not an HTTP cache — it will happily store whatever you hand it. Nothing +rejects private or authenticated responses for you. Refuse to insert when **any** holds: + +- The origin response carries `Set-Cookie`. +- The origin response is `private`, `no-store`, or `no-cache`. +- The request carried `Authorization`. +- The response is not 200 with an HTML content type. +- DataDome's request filter replaced the document. + +Audit every request-dependent rewrite before declaring the template neutral — the +integration head-inserts and the GPT-diagnostics bootstrap are both request-scoped and +must not reach C2. + +**Assert it, do not assume it.** A unit test over the transform output must fail on any +of: a bid value, an EC ID, a consent string, a geo value, a diagnostics bootstrap, or a +`Set-Cookie`. Then a second test must assert the template is **byte-identical** for two +requests differing in consent, bot classification, and prefetch status. That second test +is the one that catches this class of bug; the first would have passed on the broken +design. + +- [ ] **Step 4: Write and read C2 — with the real API** + +The builder is move-based and the insert and read handles are different objects. Naïve +code does not compile: + +```rust +// WRONG — surrogate_keys consumes the builder and returns it; this discards the +// return value and then uses a moved binding. And execute() gives a WRITE stream, +// so there is nothing to read back from it. +let mut insert = cache::core::insert(key, ttl); +insert.surrogate_keys(["ts-template"]); +let body = insert.execute()?; +``` + +Correct shape, using a transaction so a cold cache under load transforms once: + +```rust +use fastly::cache::core::{Transaction, CacheKey}; + +let tx = Transaction::lookup(CacheKey::from(key_bytes)).execute()?; + +// Order matters: a STALE entry sets BOTH found() and must_insert_or_update(). +// Testing found() first would serve the stale bytes and silently never fulfil the +// update obligation, leaving every concurrent waiter blocked until timeout. +let template: Body = if tx.must_insert_or_update() { + // Fetch and prepare BEFORE consuming `tx`. After `insert()` the transaction is + // gone and `cancel_insert_or_update()` is unreachable, so anything that can fail + // and does not need the writer belongs here. + let origin = match fetch_and_prepare_origin() { + Ok(origin) => origin, + Err(e) => { + tx.cancel_insert_or_update()?; // releases the obligation to a waiter + return fallback_uncached(e); + } + }; + + // `Transaction::insert(self)` consumes `tx` from this line on. + let (mut writer, found) = tx + .insert(template_ttl) + .surrogate_keys(["ts-template", &url_surrogate_key]) // chained, not discarded + .user_metadata(metadata_envelope) + .execute_and_stream_back()?; + + match stream_lol_html_output(origin, &mut writer) { + Ok(()) => { + writer.finish()?; // REQUIRED, and consumes `writer` + found.to_stream()? // fallible; there is no `to_body()` + } + Err(e) => { + // Also consumes `writer`, marking an unsuccessful end so no partial + // template is served. (A `StreamingBody` dropped without `finish()` is + // aborted anyway, but say it explicitly.) + writer.abandon()?; + return fallback_uncached(e); + } + } +} else if let Some(found) = tx.found() { + found.to_stream()? // C2 HIT — skip origin fetch and transform +} else { + unreachable!("a transaction is either obliged to insert or has found an item") +}; +``` + +Two ownership rules this shape exists to respect, both of which an earlier draft broke: +`Transaction::insert(self)` **consumes** the transaction, so a helper taking `&tx` cannot +call it and `cancel_insert_or_update` is unreachable afterwards; and `finish`/`abandon` +each consume the writer, so neither can be referenced from an arm that did not bind it. + +**Decide the stale policy explicitly.** `Found::is_stale()` and `is_usable()` exist, and +`stale_while_revalidate` can be set at insert. Serving stale while revalidating is a real +option — but it is a state machine, and `cache::core` implements none of it for you. The +spike should start by treating stale as a miss and only add stale-serve if the numbers +justify it. + +**`cache::core` carries no HTTP semantics.** Status, headers, content encoding, and +revalidation are all yours. Serialize what you need into `user_metadata` — at minimum the +content encoding, the transform schema version, and the origin `Vary` values the key was +built from — and decide explicitly whether the stored template is compressed. + +**Cache key must include**, beyond the origin's declared `Vary` (`rsc`, +`next-router-state-tree`, `next-router-prefetch`, `next-router-segment-prefetch`, +`Accept-Encoding` — measured, see the Stage 0 findings): + +- The full URL, explicitly. Do not rely on an ambient request key. +- **The assembly mode.** A2 and A3 emit different template bytes and would otherwise + poison each other's entries. +- **A template schema version**, bumped whenever the transform changes, so a deploy does + not read yesterday's shape. +- Request host and scheme, the enabled-integration set, and the tsjs content hash. + +Per-user signals must never appear in the key. If a signal cannot be excluded from the +template, it does not belong in C2 at all. + +### Design decided 2026-08-10: `cache::core`. Do not revisit read-through. + +An earlier revision left this open between `cache::core` and read-through caching with +`after_send` + `set_body_transform`. Investigated and verified against the pinned SDK and +Viceroy 0.17 source. **Read-through is not viable here** — not on preference, on three +hard blockers: + +1. **Viceroy stubs the entire HTTP Cache ABI**, and the SDK converts that into a _send + error_ rather than a fallback. `is_request_cacheable` returns + `Err(NotAvailable("HTTP Cache API primitives"))` + (`viceroy-lib-0.17.0/src/wiggle_abi/http_cache.rs:108-114`; 26 such stubs in that + file), which makes `must_use_host_caching()` true, which with a send hook set returns + `Err(SendErrorCause::HttpCacheApiUnsupported)` + (`fastly-0.12.1/src/http/request.rs:626-632`). **Setting `after_send` makes every + publisher origin fetch fail** under `fastly compute serve`, `cargo test-fastly`, and + the parity suite. The whole local loop dies. +2. **`with_cache_bypass` makes the hook silently dead.** `get_caching_mode` checks + `cache_override.is_pass()` **first** (`request.rs:612-615`) and returns host caching, so + `after_send` is never invoked and no error is raised. On exactly the requests in scope, + today, the hook would do nothing quietly. +3. **The closure bounds are incompatible with this codebase.** `with_after_send` requires + `Fn + Send + Sync + 'static` (`request.rs:545-550`). Everything the rewriter needs is + `!Send` by construction — `edgezero_core::body::Body` wraps a `LocalBoxStream` + deliberately, which is why the platform layer is `#[async_trait(?Send)]` throughout. + And `set_body_transform` is synchronous, so it could never await the auction collect. + +Read-through's appeal was real — `CandidateResponse::apply_and_stream_back` is +`execute_and_stream_back` with HTTP semantics attached, and TTL/SWR/vary/surrogate keys +derived from origin headers for free. It is simply unreachable from here. + +**Also settled: core cannot reach it at all.** `PlatformHttpRequest` +(`platform/http.rs:16-37`) is a plain data struct with no callback slot, and carrying one +would name `fastly::http::CandidateResponse` in portable core, breaking the other three +adapters. + +### Follow the existing null-object pattern + +`cache::core` fits the shape the repo already uses four times for a Fastly-only capability +behind a portable trait: `UnavailableHttpClient` (`platform/http.rs:216-243`), +`UnavailableKvStore` (`platform/kv.rs:14-17`), and the `RuntimeServices.kv_store` +field/accessor/builder (`platform/types.rs:170,222,269,330`). Add +`PlatformTemplateCache` the same way, and follow +`crates/trusted-server-adapter-fastly/src/ec_kv.rs` — 140 lines, the repo's only real +edge-storage read/write — rather than inventing a shape. + +**Return `EdgeBody`, not `Vec`.** `EdgeBody::Stream` exists, +`fastly_body_to_edge_stream` (`adapter-fastly/src/platform.rs:503`) already converts, and +`PublisherResponse::Buffered` tolerates a live stream (`publisher.rs:1019-1022`). + +### Exact insertion point + +**Immediately before `let mut platform_request = PlatformHttpRequest::new(...)`** — the +last line before `req` is consumed, and a few lines before the origin send. Everything +needed is in scope there: `settings`, `services`, the final URI and Host, `backend_name`, +`request_path`, `matched_slots`, `should_run_ad_stack`, `request_had_authorization`, +`request_host`, `request_scheme`. + +**One required move:** `assembly_mode` is currently computed _after_ the send, for the +logging call site. It depends only on `settings`, so hoist it above the insertion point. + +**Tee-ing is not needed.** With any post-processor registered — and the Next.js +integration always registers one — `HtmlWithPostProcessing` emits nothing until the final +chunk and then returns the whole transformed document as one contiguous buffer +(`html_processor.rs:92-97,148`). Two `write_all` calls on the same slice; no tee +abstraction, no extra copy. Still use `execute_and_stream_back`, but for transaction +correctness and request collapsing rather than for memory. On a hit the processor is never +built at all. + +- [ ] **Step 4b: close the risks the design investigation surfaced** + +Four, all specific to this codebase rather than to `cache::core` in general. + +**`Vary` is in the key list but nothing consumes it.** `c2_bypass_reason` checks +`Set-Cookie`, `Cache-Control`, `Authorization`, status and content type — **not `Vary`**. +Viceroy supports `WriteOptions.vary_rule`, so the mechanism exists; the gate has to use +it. Until then the key is missing a signal the origin explicitly declares, and Step A's +verdict is a `PROVISIONAL PASS`, not a release gate. + +**Resolved — `VarySpec`, commit `b688d667`.** Building the key exposed a problem this +plan states but does not solve: the key must cover everything the origin varies on, but +**a lookup happens before the fetch**, so on a cold key the origin's `Vary` is not yet +known. Three ways out — configure the list; two-phase lookup against a URL-keyed record +holding the last-seen `Vary`; or store the list alongside and re-key on mismatch. The +latter two are correct and double the lookups on every request. + +Configured is taken, **as a spike-grade choice rather than a production one**: Step A +already measured the origin's actual `Vary`, and a 60s TTL bounds drift to a minute +rather than indefinitely. + +The drift is guarded rather than merely accepted. `VarySpec::uncovered_by` runs _after_ +the origin responds, when its `Vary` is finally known, and names which headers the +configured spec missed. A template built under a key that did not cover something the +origin varies on **must not be stored** — a request differing only in that header would +read it. Naming the specific headers makes a stale config identifiable instead of +producing a generic refusal. + +Two decisions worth their tests. An absent header and a present-but-empty one key the +same, because the origin sees no difference between them. And `Vary: *` is not reported +as a named gap — it means uncacheable, which the eligibility gate handles, and reporting +it would produce a nonsense instruction to configure a header called `*`. + +Still open: wiring `uncovered_by` into `c2_bypass_reason` as a bypass reason, which +happens with the store call site. + +**Store bytes plus a metadata envelope; rebuild every header on a hit.** The publisher +path forces `private, no-store` and strips `ETag`/`Last-Modified`/CDN headers _after_ the +send. Replaying stored origin headers would fight that. Store only the transformed body +and a small `user_metadata` envelope — content encoding, content type, schema version, +tsjs hash — and construct every response header from scratch on a hit. Then no origin +header is ever replayed and the `Set-Cookie` privacy net is trivially safe. +`get_user_metadata` is implemented in Viceroy. + +**Content-Encoding belongs in the key.** The streaming pipeline pairs input encoding to +the same output encoding, so the transformed bytes inherit whatever the origin negotiated +from the client's `Accept-Encoding` — still gzip, deflate, br or identity after +`restrict_accept_encoding` narrows it. Either key on the negotiated encoding or normalize +to identity in the cache and re-encode on read. Getting this wrong serves brotli bytes to +a client that asked for gzip. + +**Host and scheme belong in the key.** The post-processed output is host-dependent by +construction: `request_host` and `request_scheme` reach `IntegrationHtmlContext`. + +- [ ] **Step 4c: file the wasted-dispatch follow-up** + +The auction is dispatched _before_ the insertion point. Under `Esi` and `ClientFill` the +root injects nothing, so that dispatch is already pure waste on this branch — and on a C2 +hit it is waste that must be cleaned up via `emit_abandoned_auction` or it leaks +telemetry. + +Keeping the lookup at the insertion point above is right for the spike: minimal diff, and +lookup latency overlaps the in-flight auction. Moving it earlier would eliminate the +wasted dispatch but serialize the lookup ahead of dispatch. **File it; do not fix it +here.** Suppressing root-level dispatch under the shared modes is Task 4's job, where it +also has to be reconciled with the exactly-one-auction gate. + +- [ ] **Step 5: Unit tests, then the target suite** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin assembly_mode +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +cargo fmt --all -- --check && cargo clippy-fastly +``` + +`ClientFill` must work on all four adapters. `Esi` is Fastly-only and must not break the +others' compilation. + +- [x] **Step 6: the call site — DONE.** `2db10639` (store), `2a2e6c6a` (lookup). + +The cache now engages end to end: a second request for the same URL is served without +touching the origin, and is byte-identical to what was stored. Verified by mutation — +disabling the lookup fails the hit test, so the hit is the cache answering rather than +the fixture answering twice. + +**Wiring the lookup corrected the key.** It carried the content encoding the _origin_ +chose, which does not exist at lookup time. That meant storing under `br` and looking up +under `gzip, br` — a cache that never hits. The field is now the `Accept-Encoding` sent +to the origin. Sound because negotiation is a function of what the origin was offered, +so identical offers yield identical choices; the chosen encoding stays in the metadata +and is what the served response declares. + +That made every key field request-derived, so **the key is built before the fetch** and +the response gate only authorizes storing it. A key that needed the response could only +ever authorize a store, never satisfy a read. + +**The lookup re-checks the request-derived disqualifications, and only those.** The +store gate is response-derived and cannot re-run, but need not: anything in the cache +passed it on the way in. What must re-run are properties of the _reader_ rather than of +the bytes — an authenticated request must not be served a shared template even when that +template is perfectly cacheable. + +**Shared modes take the buffered finalizer.** Storing needs every transformed byte and +streaming does not collect them. The branch keys on the store authorization rather than +on the assembly mode, so `Inline` never reaches it and the spike cannot regress the +shipped path by construction. A C2 _miss_ therefore buffers — the right trade, since a +miss is already paying an origin fetch and a full transform, and what the spike measures +is the hit, where there is no origin fetch to stream from at all. + +Every response header on a hit is constructed, never replayed, so no origin header can +reach a second visitor through the cache. + +The publisher tests use an in-memory cache double, so they prove the wiring rather than +the backing. The join they leave untested is the one `app.rs` makes: the publisher +reaches the cache as a `dyn PlatformTemplateCache` behind `RuntimeServices`, never as +the concrete type the Fastly tests exercise. That join is now executed under Viceroy +against the real Core Cache rather than only type-checked. + +**What this does not establish.** `ClientFill` and `Esi` still render a template with a +hole and nothing filling it. Task 4 and Task 5 remain the blockers on anything +deployable — a cache that works is necessary, not sufficient. + +--- + +## Task 4: Arm A2 — client-fill + +Mostly already specified. See +[the spec's Appendix B](../specs/2026-08-08-esi-cacheable-root-validation-design.md#appendix-b--stage-1-plumbing-condensed) +for the client plumbing, the two-condition join gate, and the server contract; and +[§5](../specs/2026-08-08-esi-cacheable-root-validation-design.md#5-the-trap-in-the-deferred-work--read-this-before-scheduling-stages-12) +for the silent-empty-bids trap, which applies in full. + +- [ ] **Step 1: Hoist the closure-trapped client state** — `pageBidsEndpoint`, + `requestPageBids`, and the `inflight`/`currentPath`/`lastAppliedPath` state, per + Appendix B. Do **not** route the initial load through `onNavigate`. +- [ ] **Step 2: Make `installScheduleInitialAdInit` a hydration-ready AND bids-settled + join**, with a bounded timeout that fires `adInit` untargeted rather than stranding + the slot. Derive the timeout from measured fetch latency, not a constant. +- [ ] **Step 3: Suppress the navigation-path dispatch** so exactly one auction runs per + pageview. Add a new `AuctionSource` for initial loads **plus the mechanism that + delivers it** — a header behind the same-origin gate, not a query parameter. +- [ ] **Step 4: Relocate terminal telemetry.** Navigation `Completed` is emitted only from + the collect functions; the `ts-debug` dump rides the same string. Both move. +- [ ] **Step 5: Verify exactly one auction per pageview** in `auction_events_raw`. Two is + a doubling of SSP spend and an immediate fail. + +--- + +## Task 5: Arm A3 — ESI at the edge + +- [x] **Step 0: the mechanism works — DONE.** `9539061e`, hardened in `0597f54e`. + +Verified under Viceroy with the real `esi` 0.7 crate rather than argued from docs: a +template carrying the `` seam's own ESI include tag comes back with the fragment +spliced in its place and no unresolved tag left. + +**The async/sync obstacle is dissolved, not worked around.** `esi`'s fragment dispatcher +is synchronous and this codebase's fragment producer is `async`; calling one from the +other means a nested executor, which panics. +`PendingFragmentContent::CompletedRequest` lets the dispatcher hand back an +already-built response, so the caller resolves the fragment in the normal async flow and +the dispatcher performs **no I/O at all** — no subrequest, no backend, no self-call, +nothing for Viceroy to stub. That also removes the need for a self-referencing backend +this plan would otherwise have required. + +**Step 2's instruction was right, and reading the crate showed why.** +`CacheConfig::is_includes_cacheable` defaults to **`true`**. A fragment carries one +visitor's bids, so the default caches per-user data and serves it to the next visitor — +silently, on a hit. `includes_force_ttl` is worse where set: it caches everything, +ignoring `private`, `no-store` and `Set-Cookie` alike. Both now stated explicitly, along +with `default_dca`/`inherit_parent_dca` (fragment bytes are data, never re-parsed as +ESI), `max_include_depth = 1`, and rendered caching / `edge_control` off because the +publisher path owns those headers. + +Nine tests. Four assert the configuration; the rest assert behaviour, including that a +fragment containing its own nested ESI include is spliced as text rather than dispatched, so +auction data cannot drive fragment requests. + +**What remains is the call site**, below. Emitting the include and resolving it are both +proven; connecting them is not done. + +- [ ] **Step 1: Wire `process_stream`, not the wrappers** + +`process_response` and `process_response_streaming` consume `self` _and_ send the response +themselves, which takes ownership away from the finalize / `ec_finalize` / apply-effects +ordering. `process_stream(&mut self, src: impl BufRead, out: &mut impl Write, …)` keeps it. + +Source is the C2 body. Sink is the client response body. + +**The ordering an earlier draft described is impossible.** It said EC cookie, geo, and the +privacy net run _after_ assembly. They cannot: streaming responses on this adapter +**commit headers first and then pipe chunks** +(`adapter-fastly/src/main.rs`, `send_edgezero_response`). Once ESI starts writing, no +header can change. + +The correct invariant: + +> **Finalize every header before a single body byte is written** — EC `Set-Cookie`, geo +> suppression, and an unconditional `Cache-Control: private, no-store` — **then** stream +> the assembly with no further header mutation. + +That means `private, no-store` is set unconditionally up front rather than derived from +what the assembly turns out to contain. Deriving it after the fact is not available, and +assuming it was is how a per-user response ends up shared-cacheable. + +- [ ] **Step 2: Disable DCA explicitly and allowlist the dispatcher** + +```rust +let config = esi::Configuration::default() + .with_escaped(false) + .with_default_dca(esi::DcaMode::None) // call the setter; do not rely on the default + .with_inherit_parent_dca(false); +``` + +Comments are not configuration. An earlier draft said DCA "stays at its default" — on a +pre-1.0 crate whose default could move in a patch release, and where this setting fails +**open**, that is not good enough. Call the setters. + +Also disable **fragment caching** explicitly, or mark the include `no-store="on"`. A +cached auction fragment is a per-user object in a shared cache — the C3 failure mode by +another route. + +The dispatcher must be **exact-path allowlisted**: a fragment URL that is not the bids +endpoint is refused, not fetched. The built-in dispatcher builds a dynamic backend per URL +host and panics on a hostless URL — never use it. + +Rationale in the spec's §2: bid payloads carry partner-controlled creative markup, so a +recursive parse would let an SSP make the edge fetch an arbitrary URL. **Add a unit test +that feeds a partner-controlled ESI include targeting `http://attacker.example/` through +a creative payload and asserts no fetch is attempted.** + +- [ ] **Step 3: The fragment must be a script, not the JSON endpoint** + +**`/_ts/page-bids` cannot be the ESI target.** It returns +`serde_json::json!({"slots":…, "bids":…})` (`publisher.rs:3987`), and ESI splices fragment +bytes in literally — the page would contain raw JSON where an executable script belongs. +Nothing would call `scheduleInitialAdInit`. + +Add a **dedicated fragment endpoint** returning the executable script — the same shape +`build_bids_script` produces today, plus the `adSlots` assignment that moved out of the +template in Task 3 Step 2. Either that, or use the `esi` crate's fragment-response +processor to wrap the JSON; the dedicated endpoint is simpler and easier to assert on. + +Three more things the naïve marker gets wrong: + +- **The same-origin gate will reject it.** `page_bids_request_allowed` + (`publisher.rs:3644`) requires `Sec-Fetch-Site: same-origin` or the `X-TSJS-Page-Bids` + header. An internal ESI subrequest carries neither. Give the fragment endpoint an + internal contract and a fixed backend rather than weakening that gate — it exists to + stop third parties burning SSP quota. +- **Parent context does not propagate.** EC identity, consent state, client IP, geo, User + Agent, and the correlation ID all live on the parent request. Forward an **explicitly + approved allowlist** of them into the fragment request. Forwarding everything is how a + fragment ends up more privileged than the parent. +- **Root dispatch must be suppressed.** The navigation path already dispatches an + auction. If A3 does not suppress it, every pageview runs two — doubling SSP and APS + spend. This applies to **A2 and A3 alike**. + +- [ ] **Step 4: Validate the whole URL, not the path** + +An exact-path allowlist alone permits `https://attacker.example/_ts/page-bids`. Validate +**scheme, authority, method, path, and query** — or better, ignore the marker's URL +entirely and dispatch to a fixed internal backend, treating the ESI include as a signal +rather than an address. + +Add a test that feeds an ESI include targeting +`https://attacker.example/_ts/page-bids` through a creative payload and asserts no +outbound fetch is attempted. + +- [ ] **Step 5: Deterministic synthetic fragment first** + +Before wiring the real auction, point the include at a fixed-content endpoint. This +separates "does the pipeline assemble correctly" from "does the auction behave," and the +two fail very differently. Only once assembly is proven does the fragment become the real +one. + +- [ ] **Step 6: Handle the flush hazard** + +`esi` flushes its output writer after each parse batch. Fastly's `StreamingBody` is a +`BufWriter`, so anything between esi and it must propagate `flush()` or nothing leaves the +Wasm heap. + +- [ ] **Step 7: Fragment failure must degrade, not break** + +Assert that a fragment timeout or non-2xx yields a page with empty bids rather than a 5xx +or a truncated document. Note the crate's non-obvious semantics: `alt` is attempted before +`onerror="continue"`, and `` runs **all** attempts and concatenates every +non-failed output — it is not first-success-wins. + +--- + +## Task 6: Safety gates — run against every arm + +Not a phase. Every one of these is a hard fail, independent of any performance result. + +- [x] **Zero cross-user leakage.** DONE — `76df2469`. Two synthetic users differing in EC + identity, consent jurisdiction and geo store a byte-identical template, each against + a fresh cache so the first cannot answer for the second. Forbidden-substring checks + are the second layer, since byte-identity also holds if both leak the same thing. + Mutation-verified: leaking `adSlots` through the head seam fails it. +- [x] **Cold MISS, warm HIT, stale revalidation** DONE — `76df2469`, and end to end under + `viceroy serve` (below). Stale reads as a miss; serving stale would mean serving a + template built by an older transform or bundle. + + The first stale test passed for the wrong reason and had to be rewritten: a zero TTL + produces an *absent* entry, not a stale one, so `is_stale()` was never reached — + confirmed by reverting the check and watching it stay green. Only a + `stale_while_revalidate` window makes an entry present-and-stale. + +- [x] **Transform failure** DONE — `76df2469`. A partial template in C2 is the worst + outcome available: a truncated document served to every later visitor, indefinitely, + with no error after the first request. Mutation-verified by storing before the cap + check. +- [ ] **Request collapsing** works: concurrent cold requests transform once. +- [x] **DCA disabled** DONE — `0597f54e`. Config asserted _and_ behaviour: a fragment + carrying its own nested ESI include is spliced as text rather than dispatched. + +- [ ] **Request collapsing** — not tested, and not testable here. Viceroy is + single-threaded, so the concurrent cold-request case cannot be produced. The racing + _writer_ path is covered (`a_second_put_on_a_fresh_entry_is_a_no_op`), which is the + correctness half; the collapsing half needs real concurrency. +- [ ] **Exactly one auction per pageview**, from `auction_events_raw`. +- [ ] **Cookie and privacy finalization ran BEFORE assembly**, not after — EC + `Set-Cookie` on first visit, geo suppression, and an unconditional + `Cache-Control: private, no-store`. Headers commit before the body streams on this + adapter, so "finalize after assembly" is not available; asserting it that way is how + a per-user response ends up shared-cacheable. ESI's streaming mode dropping + `$add_header` is a consequence of the same constraint, not a separate hazard. +- [ ] **Slot and bid attribution unchanged.** Same slots matched, same bids applied, same + renders attributed. Use TS-attributed renders — the SSAT line item, non-empty + `ts.bids`, `hb_adid` presence — **never slot fill**, which is blind to empty bids + because `adInit` defines slots regardless. +- [x] **No C3 — assert positively, not by absence.** DONE — `0adb578e`, and this gate's + wording caught a live bug. A C2 hit returns before the point where the publisher path + stamps `private, no-store`, so it served HTML with **no `Cache-Control` at all** — + heuristically cacheable, and therefore a shared cache of an assembled per-user + response. Checking for the _absence_ of `public`/`s-maxage`/`Surrogate-Control` would + have reported it as safe, because there was nothing present to forbid. Covered for + returning visitors specifically, where the cookie-privacy net never fires. + + Original wording, retained because it is what made the difference: Forbidding `public`, `s-maxage`, and + `Surrogate-Control` is **not sufficient**: a bare `Cache-Control: max-age=60` passes + that check and is still shared-cacheable, and that is exactly what the measured + origin sends. Require instead that every assembled response carries + `Cache-Control: private, no-store` and that `Expires`, `ETag`, `Last-Modified`, and + all four CDN cache directives are stripped. Test it for **returning** users + specifically — they set no EC cookie, so the cookie privacy net never fires and is + not a backstop here. + +--- + +## Task 7: The decision record + +**Files:** `docs/superpowers/plans/2026-08-10-1009-esi-decision-record.md` + +- [ ] **Step 1: Record every arm** with N, confidence interval, cache-tier mix, route mix, + and POP. Any arm missing those is not reportable. + +- [ ] **Step 2: Apply the decision rule, stated here before the data exists** + +**Adopt ESI only if all three hold:** + +1. Every Task 6 gate passes on A3. +2. A3 beats A2 on **bids-ready time, `adInit` fire time, and first TS-attributed creative + paint** — by a margin the reviewers ratify **before** collection, not chosen after + seeing the numbers. **Not root TTFB:** A2 and A3 serve the same C2 template, so their + root timings are near-identical by construction and a difference there would be noise. + Root TTFB is a non-regression guard only. +3. Render outcomes on A3 are non-inferior to A0. + +**Otherwise adopt A2 (client-fill)** if its gates pass and it beats A1. It is portable +across all four adapters and carries no Fastly-only maintenance burden. + +**Otherwise keep A1** — Stage 0 alone — and record #1009 as answered in the negative with +evidence. + +The margin in (2) exists because A3's cost is not its diff. It is a second rendering +architecture, Fastly-only, on a pre-1.0 crate, in the critical render path. A small win +does not pay for that. + +- [ ] **Step 3: Record what would change the answer**, so this does not get re-litigated + from scratch. At minimum: React #418 / [#938](https://github.com/IABTechLab/trusted-server/issues/938) + being fixed such that `adInit` can run synchronously, which is what would make edge + assembly's round-trip saving actually worth something. + +- [ ] **Step 4: Clean up.** Remove the spike flag or promote it to a real setting; purge + C2 (`purge_surrogate_key` on `ts-template`); remove the synthetic fragment endpoint; + and either land or delete the `esi` dependency. **A spike flag left in place becomes + permanent configuration surface.** + +--- + +## Reproducibility metadata + +Record with every result, or it cannot be re-run or trusted: commit SHA; `esi` and +`fastly` crate versions; Fastly service and version IDs; whether the backend is shielded; +`template_ttl`; the origin's `Cache-Control` and `Vary` at collection time; assembly mode; +routes; N per arm; and the cache-tier mix. + +## Out of scope + +- **Stages 1–2 of the spec** as production work. This spike may build parts of the + client-fill path to measure it; shipping it is a separate decision behind the + correctness defects. +- **Full RSC/flight partitioning.** `rsc_flight.rs` has no static/dynamic split. +- **Publisher-authored ESI.** Breaks the no-origin-changes promise. +- **A C3 delivery cache.** Not a deferred item — a thing that must not exist. + +## Definition of done + +- [ ] Task 1 verdict recorded: `esi` 0.7 builds on Rust 1.95.0 / `wasm32-wasip1`, or it + does not and the spike stopped. +- [ ] All four arms measured on one build, with correlation IDs joining server and browser + timings, and cache tier recorded per request. +- [ ] Every Task 6 gate has an explicit pass/fail per arm. +- [ ] Decision record exists, applies the pre-ratified rule, and names what would change + the answer. +- [ ] Cleanup complete: flag resolved, C2 purged, synthetic endpoint removed, dependency + landed or dropped. +- [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four + adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md new file mode 100644 index 000000000..a2b388594 --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md @@ -0,0 +1,294 @@ +# #1009 ESI Merge and Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:executing-plans` to implement +> this plan task-by-task. This plan is intentionally executed inline because the operator +> explicitly prohibited subagents. + +**Goal:** Merge current `main` and make the opt-in ESI byte-seam/shared-template path correct, +private, cache-semantic, compressed, observable, and operationally reversible. + +**Architecture:** Fastly Core Cache holds identity-encoded reader-neutral templates behind a +transaction acquired before origin work. Every request assembles its own slots and structured bid +map at an exact inert seam, encodes the result for that client, and receives a final immutable +private/no-store policy. + +**Tech Stack:** Rust 1.95, Fastly Compute/Core Cache, `edgezero_core` HTTP types, `lol_html`, +TypeScript/Vitest, Viceroy, shell harness. + +> **Implementation status, 2026-08-12:** Tasks 1–12 are complete on the branch. The Viceroy +> harness passed in both modes after running outside the filesystem sandbox so it could read the +> macOS native-certificate keychain. + +--- + +### Task 1: Merge live main and preserve auction contracts + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/integrations/gpt_diagnostics.rs` +- Modify: `crates/trusted-server-js/lib/src/core/types.ts` +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Test: adjacent Rust and Vitest modules + +- [x] Merge `origin/main` with `git merge --no-ff origin/main`. +- [x] Resolve the `AdBidsState`/`write_bids_to_state` conflict by building one structured map with + `auction_id`, storing both map and script, and returning its delivered slot IDs. +- [x] Add/adjust tests proving ESI and inline retain `hb_auction_id`, APS renderer metadata, and + delivered-winner attribution. +- [x] Run the focused Rust and GPT tests. +- [x] Complete the merge commit. + +### Task 2: Remove mechanisms outside the approved ESI byte-seam design + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Delete: `crates/trusted-server-core/src/platform/template_assembly.rs` +- Modify: `crates/trusted-server-core/src/platform/mod.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Delete: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-adapter-fastly/Cargo.toml` +- Modify: `Cargo.lock` + +- [x] Update mode tests to specify only `inline` and `esi`; watch the old client-fill expectations + fail or stop compiling. +- [x] Remove `ClientFill`, executable fragment serialization, assembler traits/registration, and + the `esi` crate. +- [x] Update comments to call the production path byte-seam assembly. +- [x] Run focused configuration, publisher, and Fastly adapter tests. +- [x] Commit the scope cleanup. + +### Task 3: Canonicalize and bound the template key + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write failing tests for absent versus empty `Vary`, repeated raw values, invalid configured + names, punctuation-colliding purge URLs, changed origin host override, and changed creative + configuration. +- [x] Replace string pairs with a typed canonical `Vary` value preserving presence and all bytes. +- [x] Hash a length-prefixed canonical key and hash the URL-specific surrogate key. +- [x] Include publisher origin identity and the complete template-shaping fingerprint. +- [x] Run focused key/configuration tests and commit. + +### Task 4: Enforce request and origin cache semantics + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/Cargo.toml` if HTTP-date parsing needs a direct dependency + +- [x] Write failing tests for response `max-age=0`, positive max age, repeated/malformed cache + directives, `Age` exhaustion, expired/malformed `Expires`, missing freshness, invalid `Vary`, + and request no-cache/no-store/range/conditional bypasses. +- [x] Add a typed cache eligibility result carrying the positive remaining TTL. +- [x] Parse relevant response directives fail-closed and cap, never extend, origin freshness. +- [x] Add request-side bypass classification before lookup. +- [x] Make unsupported/backend-failed cache lookups fall back to inline processing on non-Fastly + adapters rather than buffering a cacheless ESI path. +- [x] Run focused eligibility tests and commit. + +### Task 5: Move request collapse before the origin fetch + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` + +- [x] Verify the reservation is acquired before origin work and the Fastly transaction contract + blocks same-key waiters. Viceroy is single-threaded, so it cannot directly reproduce two + truly concurrent cold requests. +- [x] Introduce a lookup outcome with an opaque insert reservation and explicit cancellation. +- [x] Implement Fastly `Transaction::lookup` before origin work and consume/cancel its obligation + on every exit path. +- [x] Ensure invalid fresh entries become replaceable rather than causing repeated refetches. +- [x] Run focused Core Cache/Viceroy tests and commit. + +### Task 6: Make privacy and policy-header parity final + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/response_privacy.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` tests + +- [x] Write failing tests for repeated CSP/CSP-Report-Only, omitted COOP/COEP/CORP/HSTS/Link, + unknown cached header metadata, duplicate required metadata fields, and a late integration + changing `Cache-Control` to public. +- [x] Capture all ordered values, expand the safe allowlist, and decode metadata strictly. +- [x] Replay with `append`, then apply the assembled-response privacy policy last. +- [x] Preserve and reassert private/no-store after request-filter effects in Fastly's final send. +- [x] Run focused header/privacy tests and commit. + +### Task 7: Bypass shared templates for request-private diagnostics + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write a failing warm-cache test activated by diagnostics query and another by diagnostics + cookie. +- [x] Make `requires_private_no_store()` a lookup/store disqualifier. +- [x] Verify ordinary diagnostics-disabled requests still hit C2. +- [x] Run focused diagnostics/C2 tests and commit. + +### Task 8: Re-encode assembled responses for the reader + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write failing cold/warm tests requiring gzip/br clients to receive a matching encoded body + and proving reader encoding no longer partitions the stored template. +- [x] Keep the origin offer within the reader's supported codings so a response-gate bypass remains + lossless, while decoding every stored template to identity. +- [x] Carry the selected response encoding separately from identity template metadata. +- [x] Encode buffered assembly after splicing and stream hit prefix/seam/suffix through one encoder. +- [x] Handle `identity;q=0` without serving an unacceptable representation. +- [x] Emit the correct `Vary: Accept-Encoding` response semantics after final encoding. +- [x] Run focused compression tests and commit. + +### Task 9: Make marker failures safe + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` + +- [x] Write failing tests for HTML with no explicit ``, a publisher-authored marker + collision, and a corrupt cached marker. +- [x] Record/validate a schema-bound seam location or use a collision-resistant marker contract. +- [x] Cancel storage and fall back safely when the optimization cannot produce one seam. +- [x] Run focused miss/hit assembly tests and commit. + +### Task 10: Add operational observability and harden the harness + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` +- Modify: `scripts/c2-local-test.sh` +- Modify: `.github/workflows/test.yml` + +- [x] Write failing tests for distinct backend-error versus not-found status and C2 response-state + reporting. +- [x] Preserve backend errors and emit bounded C2 status without exposing key material. +- [x] Change the harness to operate on a temporary manifest and fail on missing/non-numeric probe + output or empty response bodies. +- [x] Test both cold and warm integrity and execute the generated scheduler payload contract. +- [x] Add the ESI harness to CI where Viceroy prerequisites are available. +- [x] Run shell syntax/static checks and commit. + +### Task 11: Document configuration, semantics, and rollback + +**Files:** + +- Modify: `trusted-server.example.toml` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md` +- Modify: `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md` +- Modify: relevant #1009 findings documents + +- [x] Document that `esi` means Fastly C2 plus byte-seam assembly, not parser execution or final + HTTP shared caching. +- [x] Document `template_cache_vary`, cookie independence, freshness, metrics, purge, rollback + ordering, and limitations on non-Fastly adapters. +- [x] Close or supersede stale spike checkboxes and remove claims contradicted by the final code. +- [x] Run docs format/build and commit. + +### Task 12: Full verification + +**Files:** none expected beyond fixes discovered by verification + +- [x] Run `cargo fmt --all -- --check`. +- [x] Run all four adapter test aliases and the parity suite. +- [x] Run all six clippy aliases. +- [x] Build the Fastly release WASM. +- [x] Run JS tests, build, and format under pinned Node 24.12.0. +- [x] Run docs format/build. +- [x] Run `scripts/c2-local-test.sh esi` and `inline` if the environment exposes the required + local certificate store; otherwise report the exact environment blocker. +- [x] Run `git diff --check`, inspect the merge graph, and confirm the worktree contains only + intended changes. + +### Task 13: Interpret Fastly Surrogate-Control conservatively + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Write a failing gate test using `Cache-Control: max-age=60` plus the observed publisher + `Surrogate-Control` policy (`max-age=1200`, `stale-while-revalidate=21600`, and + `stale-if-error=604800`). +- [x] Write failing tests proving the shorter standard/surrogate freshness wins, stale windows do + not extend fresh reuse, restrictive directives are refused, and unknown, duplicate, or + malformed directives fail closed. +- [x] Parse only Fastly's supported `max-age`, `stale-while-revalidate`, and `stale-if-error` + directives; continue refusing every other vendor CDN policy field. +- [x] Keep request `Cache-Control: max-age=0` as an intentional C2 bypass so reload preserves its + revalidation semantics. +- [x] Run focused tests, `cargo test-fastly`, target-matched formatting/clippy, both local harness + modes, and verify the observed publisher policy progresses from `miss-stored` to `hit` in + the local Fastly runtime on an ordinary navigation. + +### Task 14: Allow browser reloads to reuse a fresh ESI template + +Task 14 supersedes Task 13's conservative request `max-age=0` bypass after end-to-end testing +proved that C2 reuses only the neutral template and still creates a new private response and +auction. + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Write a failing end-to-end test proving `Cache-Control: max-age=0` reruns the auction but + does not refetch the reader-neutral publisher template. +- [x] Treat only a valid zero request max age as compatible with C2; continue bypassing positive + or malformed constraints and every explicit revalidation directive. +- [x] Verify the focused tests, formatting, and Fastly clippy, then commit independently. + +### Task 15: Make the ESI template-cache ceiling configurable + +Task 15 supersedes Task 13's shorter-of-standard-and-surrogate rule. The final behavior follows +Fastly edge precedence while retaining restrictive directives as hard refusals. + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `trusted-server.example.toml` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Add failing configuration tests for the 60-second default, an explicit 1,200-second ceiling, + zero, values above one day, and omission from serialized rollback-compatible config. +- [x] Add failing freshness tests proving Fastly precedence, age deduction, and the configured + ceiling for the observed `Cache-Control: max-age=60` plus + `Surrogate-Control: max-age=1200` response. +- [x] Implement `template_cache_max_age_seconds` under `[creative_opportunities]` and thread its + resolved duration into C2 eligibility. +- [x] Remove the Fastly adapter's second hard-coded 60-second cap; the already-authorized + per-entry max age becomes the sole insertion lifetime. +- [x] Update the example and operator guide, without editing the tracked deployment + `fastly.toml`. +- [x] Run focused red/green tests, full adapter tests and clippy gates, documentation checks, and + inspect the final diff with `fastly.toml` excluded. diff --git a/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md b/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md new file mode 100644 index 000000000..a4c04b9f8 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md @@ -0,0 +1,104 @@ +# #1009 ESI Parser Assembly Implementation Plan + +> **Execution note:** Implemented inline in the current checkout, without a worktree or +> subagents, as requested. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Use the repaired ESI parser on authorized cold C2 misses without changing the existing warm-hit streaming behavior. + +**Architecture:** C2 retains the inert schema-v4 seam. Core delegates cold assembly through a platform trait; Fastly converts the seam to one synthetic ESI include and resolves it from the already-collected per-reader script. Parser failure falls back to core's validated byte split, while warm hits continue to stream by byte seam. + +**Tech Stack:** Rust 1.95, `wasm32-wasip1`, Fastly Compute/Viceroy, `stackpop/esi` pinned by Git revision, `error-stack`. + +--- + +### Task 1: Restore a platform assembly boundary + +**Files:** + +- Create: `crates/trusted-server-core/src/platform/template_assembly.rs` +- Modify: `crates/trusted-server-core/src/platform/mod.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Test: `crates/trusted-server-core/src/platform/template_assembly.rs` + +- [x] Add a failing object-safety/default-behavior test for `PlatformTemplateAssembler`. +- [x] Run the focused core test and confirm it fails because the boundary is absent. +- [x] Add the trait, error type, unavailable default, runtime service field, builder method, + accessor, and test support. +- [x] Run the focused tests and confirm they pass. + +### Task 2: Delegate only cold-miss assembly + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` + +- [x] Add a recording assembler to the C2 end-to-end tests. +- [x] Add a test asserting one platform call on a cold miss and no additional call on the + subsequent warm hit. +- [x] Add a test asserting platform failure returns a complete byte-seam response. +- [x] Add tests for `x-ts-assembly` values on parser, fallback, and warm paths. +- [x] Run each test first and confirm the expected failure. +- [x] Change `assemble_if_shared` to call the platform assembler after storage, fall back + to the validated byte split on error, and return the assembly method. +- [x] Set `x-ts-assembly` without changing `x-ts-c2-cache` or privacy headers. +- [x] Re-run the C2 end-to-end test module. + +### Task 3: Add the repaired Fastly ESI adapter + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/Cargo.toml` +- Modify: `Cargo.lock` +- Create: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Test: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` + +- [x] Add failing adapter tests for a large Next.js script followed by the seam, an + unexpected publisher ESI directive, an unexpected dispatcher URL, and verbatim + fragment content. +- [x] Run the focused Fastly test filter and confirm the missing module/implementation + fails. +- [x] Pin `https://github.com/stackpop/esi.git` at + `4c53feab4d22ad9a84641b4c46f3f63bc6d197e2`. +- [x] Implement the explicit no-cache/no-DCA ESI configuration and synthetic completed + fragment dispatcher. +- [x] Register `FastlyTemplateAssembler` in per-request runtime services. +- [x] Run the focused Fastly tests and confirm they pass. + +### Task 4: Preserve cache schema and documentation truth + +**Files:** + +- Modify: `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md` +- Modify: `docs/guide/configuration.md` +- Modify: `scripts/c2-local-test.sh` +- Test: `crates/trusted-server-core/src/platform/template_cache.rs` + +- [x] Add/adjust tests proving schema version 4 and the inert stored marker remain + unchanged. +- [x] Extend the local harness to require `esi-parser` on the miss and `byte-seam` on the + hit. +- [x] Update architecture and operator documentation to describe the hybrid path and + pinned fork accurately. +- [x] Run formatting and the harness's static checks. + +### Task 5: Full verification and signed commit + +**Files:** + +- Review every modified file. + +- [x] Run `cargo fmt --all -- --check`. +- [x] Run every target-matched Clippy alias from `CLAUDE.md`. +- [x] Run `cargo test-fastly`, `cargo test-axum`, `cargo test-cloudflare`, and + `cargo test-spin`. +- [x] Run the integration parity test. +- [x] Run JS tests/build/format and docs format. +- [x] Run the C2 local harness when Viceroy and its certificate environment are + available; otherwise report that environmental gap explicitly. +- [x] Run `git diff --check`, inspect staged scope, and confirm no operator configuration + or secrets are staged. +- [x] Create one SSH-signed commit only after every required gate is green. +- [x] Verify the commit signature locally and report the exact commit ID and test counts. diff --git a/docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md b/docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md new file mode 100644 index 000000000..bbe214363 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-ssat-debug-comment-format.md @@ -0,0 +1,232 @@ +# SSAT Debug Comment Output Format Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a backward-compatible `compact`/`pretty` configuration option for the outer JSON in SSAT auction debug comments. + +**Architecture:** Extend `AuctionDebugCommentOptions` with a serde-backed format enum that defaults to compact. Keep the existing dump value and safety pipeline unchanged, selecting only `serde_json::to_string` versus `serde_json::to_string_pretty` before terminator neutralization and the 256 KiB cap. + +**Tech Stack:** Rust 2024, serde, serde_json, TOML configuration, existing `trusted-server-core` unit tests. + +--- + +## File Map + +- Modify `crates/trusted-server-core/src/settings.rs`: define the output-format enum, add the option and default, and test TOML behavior. +- Modify `crates/trusted-server-core/src/publisher.rs`: select compact or pretty JSON serialization and test rendering/safety invariants. +- Modify `trusted-server.example.toml`: document the new setting and accepted values. + +### Task 1: Add the configuration type + +**Files:** + +- Modify: `crates/trusted-server-core/src/settings.rs:1945-2030` +- Test: `crates/trusted-server-core/src/settings.rs:2710-2780` + +- [ ] **Step 1: Write failing settings tests** + +Extend `auction_debug_comment_options_default_matches_serde_defaults` with: + +```rust +assert_eq!( + opts.format, + AuctionDebugCommentFormat::Compact, + "should default to compact output" +); +``` + +Add focused tests: + +```rust +#[test] +fn auction_debug_comment_options_deserializes_pretty_format() { + let options: AuctionDebugCommentOptions = toml::from_str(r#"format = "pretty""#) + .expect("should deserialize pretty format"); + assert_eq!(options.format, AuctionDebugCommentFormat::Pretty); +} + +#[test] +fn auction_debug_comment_options_bad_format_fails_config_load() { + let result: Result = + toml::from_str(r#"format = "expanded""#); + assert!( + result.is_err(), + "unrecognized format must fail to deserialize, not silently fall back" + ); +} +``` + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +cargo test-fastly auction_debug_comment_options -- --nocapture +``` + +Expected: compilation fails because `AuctionDebugCommentFormat` and `format` do not exist. + +- [ ] **Step 3: Implement the minimal configuration surface** + +Add `format` after `verbosity` in `AuctionDebugCommentOptions`: + +```rust +/// JSON representation used for the outer auction dump. +#[serde(default)] +pub format: AuctionDebugCommentFormat, +``` + +Set `format: AuctionDebugCommentFormat::Compact` in the hand-written default and define: + +```rust +/// JSON representation used for the outer `ts-debug` auction dump. +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuctionDebugCommentFormat { + #[default] + Compact, + Pretty, +} +``` + +- [ ] **Step 4: Run the focused tests and verify GREEN** + +Run `cargo test-fastly auction_debug_comment_options -- --nocapture`. + +Expected: all matching settings tests pass. + +- [ ] **Step 5: Commit the configuration change** + +```bash +git add crates/trusted-server-core/src/settings.rs +git commit -m "Configure SSAT debug comment output format" +``` + +### Task 2: Render pretty outer JSON without transforming values + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs:2027-2130` +- Test: `crates/trusted-server-core/src/publisher.rs:4330-4945` + +- [ ] **Step 1: Add a test helper for extracting an uncapped dump** + +Extract the existing parsing logic into `dump_json_from_comment`, returning the dump substring and parsed `serde_json::Value`. It must split after `dump=` and before the comment's final newline/terminator so it works for both one-line and indented JSON. + +- [ ] **Step 2: Write failing rendering tests** + +Add tests proving: + +1. Default compact output still contains `dump={"provider_responses":` and no newline immediately after the opening object. +2. Pretty output contains `dump={\n "provider_responses":`. +3. Compact and pretty uncapped dumps deserialize to equal JSON values. +4. In Full mode, metadata containing `{"requestbody": "{\"id\":\"request-1\"}"}` retains `requestbody` as a JSON string in pretty output. + +Extend the existing comment-terminator test to iterate over both +`AuctionDebugCommentFormat::{Compact, Pretty}` for the tested verbosity modes. +Add a pretty/full case to the total-cap test and assert the existing +`(truncated` marker remains present. + +- [ ] **Step 3: Run the rendering tests and verify RED** + +Run: + +```bash +cargo test-fastly auction_debug_comment -- --nocapture +``` + +Expected: the pretty-layout assertion fails because rendering still always uses compact serialization. + +- [ ] **Step 4: Implement format-selected serialization** + +Import `AuctionDebugCommentFormat` alongside the existing options types. Replace the single serializer call with: + +```rust +let serialized = match options.format { + AuctionDebugCommentFormat::Compact => serde_json::to_string(&dump), + AuctionDebugCommentFormat::Pretty => serde_json::to_string_pretty(&dump), +}; +let dump = render_dump( + serialized.unwrap_or_else(|error| format!("")), +); +``` + +Do not alter the dump value, nested strings, neutralization, cap, or comment envelope. + +- [ ] **Step 5: Run the focused tests and verify GREEN** + +Run `cargo test-fastly auction_debug_comment -- --nocapture`. + +Expected: all matching rendering tests pass. + +- [ ] **Step 6: Commit the renderer change** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Pretty print SSAT debug comment dumps" +``` + +### Task 3: Document and verify the completed feature + +**Files:** + +- Modify: `trusted-server.example.toml:180-190` + +- [ ] **Step 1: Document the setting** + +Add to `[debug.auction_html_comment_options]`: + +```toml +# "compact" (default) or "pretty". Pretty formats only the outer dump; +# JSON request/response bodies remain strings exactly as captured. +format = "compact" +``` + +- [ ] **Step 2: Run formatting** + +Run `cargo fmt --all` and then `cargo fmt --all -- --check`. + +Expected: both exit successfully. + +- [ ] **Step 3: Run required tests** + +Run each command separately: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: every command exits successfully with zero failed tests. + +- [ ] **Step 4: Run target-matched lint checks** + +Run each command separately: + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: every command exits successfully with warnings denied. + +- [ ] **Step 5: Inspect the final diff and configuration compatibility** + +Run `git diff --check` and inspect `git diff origin/main...HEAD` plus the remaining working-tree diff. Confirm compact is the serde/default value, pretty changes whitespace only, nested strings are preserved, and both safety protections remain unconditional. + +- [ ] **Step 6: Commit documentation and any formatting changes** + +```bash +git add crates/trusted-server-core/src/settings.rs crates/trusted-server-core/src/publisher.rs trusted-server.example.toml +git commit -m "Document SSAT debug comment formatting" +``` + +- [ ] **Step 7: Push the branch and update PR #943** + +Run `git push origin feat/ssat-debug-comment-config`, then verify PR state and checks with `gh pr view 943` and `gh pr checks 943`. diff --git a/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md new file mode 100644 index 000000000..018f7f685 --- /dev/null +++ b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md @@ -0,0 +1,848 @@ +# Server-Side Ad Template CLI Design + +**Date:** 2026-06-26 +**Status:** Draft design +**Scope:** Static and browser-backed CLI diagnostics for server-side ad templates + +## 1. Goal + +Add Trusted Server CLI support for server-side ad-template onboarding and +verification without resurrecting the stale standalone `ts-config` design. + +The CLI must answer two operator questions: + +1. Given an effective `trusted-server.toml`, which configured ad-template slots + match this path? +2. Given one or more live publisher URLs, are the configured slots for the + final navigated paths actually present on the page according to DOM, GPT, + and provider evidence, and do any runtime gates explain why Trusted Server + would not inject or auction for that page? + +The command surface is split by whether the command is local-config-only or +browser-backed: + +```bash +ts config ad-templates lint +ts config ad-templates match +ts config ad-templates check +ts config ad-templates explain + +ts audit ad-templates verify ... +``` + +Static commands live under `ts config` because they only load local effective +app config. Browser-backed verification lives under `ts audit` because it loads +public publisher pages in Chrome/Chromium and observes live page behavior. + +## 2. Context + +This design replaces the stale PR #724 direction. + +PR #724 designed a standalone `ts-config` binary around a +`creative-opportunities.toml` file. That is no longer the project shape: + +- Trusted Server configuration now flows through the unified `ts` CLI from PR + #799. +- Server-side ad-template slots live under `[creative_opportunities]` / + `[[creative_opportunities.slot]]` in `trusted-server.toml`. +- Effective config can include EdgeZero app-config environment overlays unless + `--no-env` is passed. +- Operator-owned `trusted-server.toml` is ignored; the repository tracks + `trusted-server.example.toml`. + +PR #799 is the CLI base. It owns the `ts` binary, EdgeZero lifecycle delegates, +and typed app-config validation/push/diff behavior. + +PR #800 is the audit dependency. It adds the generic browser-backed +`ts audit ` collector using local Chrome/Chromium. At the time this spec +was written, PR #800 was stale relative to the latest #799 head, so this work +depends on the #800 audit collector after it is rebased onto the latest #799 +typed blob-config model. + +## 3. Non-Goals + +- Do not add a standalone `ts-config` binary. +- Do not reintroduce `creative-opportunities.toml`. +- Do not implement browser-backed generation in Phase 1. +- Do not mutate `trusted-server.toml` from `verify`. +- Do not probe PBS, GAM, or APS management APIs. +- Do not require EdgeZero platform adapters for local static diagnostics. +- Do not make `ts audit ad-templates verify` push, provision, deploy, or update + platform resources. +- Do not rely on real GPT or APS network calls in tests. + +Browser-backed generation is a later phase: + +```bash +ts audit ad-templates generate ... +``` + +That phase needs separate rules for slot ID derivation, page-pattern inference, +multi-URL merging, TOML ordering, and whether the command emits a patch, a draft +file, or full config blocks. + +## 4. Command Surface + +### 4.1 Shared Config Flags + +All `ts config ad-templates ...` commands and +`ts audit ad-templates verify` accept the same local app-config flags: + +```bash +--app-config +--manifest +--no-env +``` + +Defaults match PR #799: + +| Option | Default | +| -------------- | ------------------------------------------------ | +| `--app-config` | `.toml`, resolved from `edgezero.toml` | +| `--manifest` | `edgezero.toml` | +| `--no-env` | `false`; app-config env overlay is applied | + +If an explicit `--app-config` path is supplied and missing, the command reports +that path as the error. It must not silently fall back to an environment or +manifest-derived path. + +### 4.2 Static Config Diagnostics + +```bash +ts config ad-templates lint [--app-config ] [--manifest ] [--no-env] +``` + +Reports whether `[creative_opportunities]` is configured, how many slots exist, +GAM network ID, auction timeout, auction enablement, configured auction +providers, and whether current EdgeZero routing will fall back to the legacy +path when configured slots are present. + +```bash +ts config ad-templates match [--details] ... +``` + +Normalizes a path or full URL to a path and reports the slots matched by the +runtime `creative_opportunities::match_slots` logic. `--details` includes slot +div ID, GAM unit path, page patterns, formats, and configured providers. + +```bash +ts config ad-templates check \ + (--expected-slot ... | --expect-no-slots) \ + [--allow-extra-slots] ... +``` + +CI-friendly assertion wrapper around the same matching logic. + +```bash +ts config ad-templates explain \ + [--method GET] \ + [--non-navigation] \ + [--prefetch] \ + [--bot] \ + [--consent-denied] \ + [--edgezero-enabled] ... +``` + +Explains the major runtime gates that decide whether the server-side ad stack +would run for a page request. This is a local model, not a live request replay. + +### 4.3 Browser-Backed Verification + +```bash +ts audit ad-templates verify ... \ + [--app-config ] \ + [--manifest ] \ + [--no-env] \ + [--strict] \ + [--json] \ + [--scroll] +``` + +Behavior: + +- Accept one or more `http` or `https` URLs. +- Reject all other schemes before launching a browser. +- Load the effective Trusted Server app config. +- For each URL, navigate first, collect the final URL, normalize the final URL + to a path, and call `creative_opportunities::match_slots`. +- Preserve the requested URL/path separately from the final URL/path. +- Emit a redirect warning when the final path differs from the requested path. +- Expect only the slots matched for the final URL path to be present on that + live page. +- Report live DOM/GPT/APS ad-slot evidence that does not correspond to a + matched configured slot as structured extra evidence. +- Launch Chrome/Chromium through the audit collector from the rebased #800 work. +- Inject a read-only ad-template collector before publisher scripts run. +- Compare configured matched slots against DOM, GPT, and APS evidence. +- Report runtime ad-stack gate evidence separately from placement evidence. +- Print human output by default. +- Emit stable machine-readable output with `--json`. +- Exit `0` by default for missing or partial live evidence; this is an + auditor-assist mode. +- Exit non-zero under `--strict` when a matched configured slot is missing or + only partially confirmed. + +`--scroll` performs a deterministic scroll pass after initial load and settle. +It is opt-in because it is slower and can trigger additional page behavior. +Slots first observed during scroll count as confirmed when the GPT evidence is +otherwise sufficient. + +## 5. Confirmation Model + +The verifier compares configured expected slots to live page evidence. + +It must keep three concepts separate: + +1. **Static slot matching:** which configured slots match a URL path according + to `creative_opportunities::match_slots`. +2. **Runtime ad-stack eligibility:** whether Trusted Server would run its + server-side ad stack for the audited navigation. This mirrors + `should_run_server_side_ad_stack` for the initial publisher request and the + `/__ts/page-bids` kill-switch/consent behavior for SPA route updates. +3. **Live placement evidence:** what the browser actually observes on the + rendered page through DOM, GPT, and APS evidence. + +`verify` is primarily a live placement verifier. `--strict` fails when matched +configured slots for an eligible page are missing or partial. Runtime gates are +reported so operators can distinguish "the slot is not on the page" from "the +current request/config would intentionally suppress Trusted Server ad-template +injection or page-bids slot output". + +### 5.1 Expected Slots + +For each input URL: + +1. Navigate the browser to the requested URL. +2. Record `requested_url`, `requested_path`, `final_url`, and `final_path`. +3. Match configured slots through the core runtime matcher using `final_path`. +4. Build an expected-slot record for each matched slot: + - slot ID; + - resolved div ID; + - resolved GAM unit path; + - configured formats; + - configured providers; + - matching page patterns. + +Only these expected slots are verified for that page. For example, slots whose +only pattern is `/` are expected for the homepage path, not for `/news/story`. + +When a navigation redirects, `verify` uses the final path for expected slots and +reports the requested path in output. This matches runtime behavior: Trusted +Server evaluates the actual publisher request path it handles, not the URL the +operator typed before redirects. + +### 5.2 Runtime Gate Evidence + +For each page result, `verify` reports a local runtime-gate model: + +| Gate | Source | +| ------------------------ | -------------------------------------------------------------------------------------------------------- | +| `method_get` | Browser navigation request; expected to pass for normal `verify`. | +| `navigation` | Browser navigation request; expected to pass for normal `verify`. | +| `not_prefetch` | Browser request headers; expected to pass unless the collector is extended with prefetch simulation. | +| `not_bot` | Browser User-Agent checked against the runtime bot fragments. | +| `matched_slots` | Final-path slot matching. | +| `auction_enabled` | Effective `[auction].enabled` / orchestrator enablement from app config. | +| `consent_allows_auction` | `unknown` unless the collector can prove a consent-allowed or consent-denied state for the live request. | + +`runtime_ad_stack_expected` is a three-state value: `yes`, `no`, or `unknown`. +Known blocking gates produce page warnings and set +`runtime_ad_stack_expected = "no"`. Unknown gates set +`runtime_ad_stack_expected = "unknown"` but do not by themselves fail +`--strict`. + +If `runtime_ad_stack_expected = "no"` because of a known config/request gate +such as `[auction].enabled = false`, strict mode does not fail missing GPT/APS +evidence for that page. The page result is reported as skipped for runtime +verification while still showing the static expected slots and any live +placement evidence that was observed. + +If `runtime_ad_stack_expected = "yes"` or `"unknown"`, strict mode applies the +normal missing/partial placement rules from §5.6. + +For SPA routes, `/__ts/page-bids` returns no slots when the ad-stack kill switch +or consent gate blocks the stack. Browser verification should report observed +page-bids responses when available, but it must not require real partner bids in +tests. + +Live ad-slot evidence that does not map to a matched expected slot is reported +as structured extra evidence. Extra evidence can identify publisher-owned slots +that have not yet moved into server-side ad templates, slots whose +`page_patterns` are too narrow, or slots that should stay outside Trusted +Server. It does not make `--strict` fail in Phase 1. + +### 5.3 DOM Slot Resolution + +The verifier must mirror the runtime GPT bootstrap's slot-root resolution: + +1. Try `document.getElementById(slot.div_id)`. +2. If absent, find the first element with an ID that starts with `slot.div_id`. +3. Ignore elements whose ID ends with `-container`. + +This is required because `div_id` may intentionally be a stable prefix for +framework-generated IDs, for example `ad-header-0-`. + +### 5.4 GPT Evidence + +A slot is confirmed by GPT evidence when the live page exposes a GPT slot whose: + +- ad unit path equals the configured resolved GAM unit path; +- slot element ID equals the resolved DOM element ID or an existing + `${resolved_dom_id}-container` element used by Trusted Server when defining + its own slot; +- configured sizes are compatible with the observed GPT sizes. + +The collector should observe both direct `googletag.defineSlot` calls and +post-load `googletag.pubads().getSlots()` state. + +Size compatibility is defined for Phase 1 as follows: + +- Normalize configured sizes from `CreativeOpportunityFormat` values where + `media_type = "banner"` into `(width, height)` pairs. +- Normalize observed GPT sizes from `defineSlot` input and `getSizes()` output: + - `[300, 250]` becomes one `(300, 250)` pair. + - `[[300, 250], [728, 90]]` becomes two pairs. + - non-numeric values such as `"fluid"` are ignored for numeric matching and + reported as warnings. +- A GPT slot's sizes are compatible when the configured banner size set and the + observed numeric GPT size set have at least one pair in common. +- Extra observed GPT sizes do not block confirmation, but they are reported as + warnings so auditors can decide whether to add formats to config. +- Configured banner sizes that are not observed do not block confirmation when + at least one configured size was observed, but they are reported as warnings. +- If ad unit path and div match but no numeric size overlap exists, the slot is + `partial`, not `confirmed`. +- Configured `video` and `native` formats are not used for Phase 1 GPT size + confirmation. If a matched slot has only non-banner formats, the verifier + reports it as `partial` with an unsupported-format warning unless a later + phase defines video/native verification. +- Out-of-page GPT slots are not confirmed in Phase 1 because the current + server-side ad-template path is slot/div based. They are reported as warnings + when observed. + +### 5.5 APS Evidence + +When a configured slot has `providers.aps.slot_id`, the collector records +`apstag.fetchBids` calls and compares configured slot IDs and sizes with the APS +payload. + +APS evidence is a provider-level signal. Missing or ambiguous APS evidence +creates a provider warning, but it does not by itself make an otherwise GPT- +confirmed slot fail `--strict` in Phase 1. + +### 5.6 Statuses + +| Status | Meaning | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `confirmed` | GPT evidence matches the configured GAM unit path, div resolution, and compatible sizes. | +| `partial` | The page has some evidence for the configured slot, but not enough to confirm it. This includes DOM-only evidence, GPT path/div matches with incompatible sizes, GPT path/div matches for unsupported non-banner-only configured formats, and other non-confirming GPT evidence. | +| `missing` | No DOM or GPT evidence confirms the configured slot. | + +In `--strict` mode: + +- `missing` fails. +- `partial` fails. + +Provider issues are not statuses. They are warnings attached to the slot result. +For example, a slot can be `confirmed` and still carry a warning that configured +APS evidence was missing or ambiguous. Provider warnings do not fail `--strict` +unless a future `--strict-providers` flag is added. + +## 6. Architecture + +The architecture should keep command parsing thin and move ad-template behavior +into pure, testable modules. + +```text +crates/trusted-server-cli/src/ + app_config.rs + ad_templates/ + mod.rs + expected.rs + compare.rs + output.rs + config_ad_templates.rs + audit/ + page.rs + browser.rs + ad_templates.rs +``` + +### 6.1 `app_config.rs` + +Shared loader for effective Trusted Server app config. + +Responsibilities: + +- read `edgezero.toml` through EdgeZero manifest helpers; +- resolve the default `.toml` path; +- apply EdgeZero app-config env overlay unless `--no-env`; +- return `TrustedServerAppConfig` / `Settings`; +- report errors in the same terms as #799 config commands. + +This avoids duplicating config path and env-overlay behavior between +`ts config ad-templates ...` and `ts audit ad-templates verify`. + +The current branch already has a private loader in `config_ad_templates.rs`. +Before adding browser-backed verification, move that behavior into this shared +module and route the existing static commands through it so both command +families load the same effective config. + +### 6.2 `ad_templates::expected` + +Pure local expected-slot model. + +Responsibilities: + +- normalize path-or-URL input; +- call `creative_opportunities::match_slots`; +- convert matched slots into stable expected-slot structs; +- preserve deterministic ordering by slot order from config. + +This module must not compile glob patterns independently or duplicate matching +semantics. + +If richer pattern diagnostics are needed, add a small helper to +`trusted-server-core::creative_opportunities` and use it from both runtime and +CLI. + +### 6.3 `ad_templates::compare` + +Pure comparison between expected slots and collected browser evidence. + +Responsibilities: + +- implement DOM prefix matching rules; +- compare GPT path, div, and size evidence; +- compare APS evidence; +- collect unmatched live DOM/GPT/APS ad-slot evidence as structured + `extra_evidence`; +- assign `confirmed`, `partial`, `missing`, and provider warning details; +- decide strict failure status. + +This module should be testable without launching Chrome. + +### 6.4 `ad_templates::output` + +Human and JSON output model. + +Responsibilities: + +- serialize stable JSON output; +- keep arrays ordered by input URL, then configured slot order, then provider + name; +- render concise human summaries; +- avoid leaking page HTML, cookies, local storage, or arbitrary page data. + +### 6.5 `config_ad_templates.rs` + +Thin Clap adapter for `ts config ad-templates ...`. + +Responsibilities: + +- parse command arguments; +- call `app_config` and `ad_templates::expected`; +- delegate formatting to `ad_templates::output`; +- keep no browser-specific logic. + +### 6.6 `audit::browser` + +Shared browser utility extracted from or aligned with the rebased #800 audit +collector. + +Responsibilities: + +- locate Chrome/Chromium; +- launch an isolated profile; +- reject non-HTTP(S) URLs before navigation; +- set bounded navigation and settle timeouts; +- run optional init scripts; +- perform optional deterministic scroll; +- collect final URL, title, rendered scripts, resource entries, and optional + ad-template evidence. + +The generic `ts audit ` command from #800 should continue to work without +ad-template verification enabled. + +### 6.7 `audit::ad_templates` + +Browser-backed verifier orchestration. + +Responsibilities: + +- parse `ts audit ad-templates verify`; +- load effective config through `app_config`; +- compute expected slots for each URL; +- run the browser collector with ad-template evidence enabled; +- call `ad_templates::compare`; +- print human or JSON output; +- apply default auditor-assist exit behavior and `--strict` behavior. + +## 7. Browser Collector + +The ad-template collector is injected before page scripts run. It is read-only: +it records evidence and calls original page functions with unchanged arguments. + +The rebased #800 collector must grow a pre-navigation init-script hook before it +can satisfy this spec. The stale #800 collector only navigates, waits, and reads +post-load page state; that is insufficient for GPT/APS call evidence. + +Instrumentation requirements: + +- install the collector through the browser's "evaluate on new document" / + init-script mechanism before navigation; +- serialize only configured div prefixes and provider IDs needed for matching; +- observe pages that create `window.googletag = { cmd: [] }` after injection; +- wrap `googletag.cmd.push` callbacks without changing callback order; +- record direct `googletag.defineSlot` calls and calls executed from the GPT + command queue; +- read final `googletag.pubads().getSlots()` state after settle and after + scroll; +- observe pages that assign `window.apstag` after injection and wrap + `apstag.fetchBids` when present; +- tolerate pages that never load GPT or APS and report warnings instead of + throwing collector errors. + +Evidence to collect: + +- DOM elements with IDs relevant to configured slot div prefixes; +- calls to `googletag.defineSlot`; +- final `googletag.pubads().getSlots()` state after settle and after scroll; +- calls to `apstag.fetchBids`; +- timestamps or phases indicating whether evidence was observed during + `initial_load` or `scroll`. + +The collector must not: + +- block, rewrite, or suppress publisher scripts; +- override `navigator.webdriver`; +- capture cookies, local storage, session storage, request bodies, or arbitrary + page data; +- require real GPT/APS network calls in test fixtures. + +## 8. JSON Output Contract + +`--json` emits deterministic JSON. Shape: + +```json +{ + "ok": true, + "strict": false, + "pages": [ + { + "url": "https://www.example.com/news/story", + "final_url": "https://www.example.com/news/story", + "requested_path": "/news/story", + "path": "/news/story", + "runtime_ad_stack_expected": "unknown", + "gates": { + "method_get": "pass", + "navigation": "pass", + "not_prefetch": "pass", + "not_bot": "pass", + "matched_slots": "pass", + "auction_enabled": "pass", + "consent_allows_auction": "unknown" + }, + "matched_slot_count": 1, + "slots": [ + { + "id": "atf", + "status": "confirmed", + "phase": "initial_load", + "configured": { + "div_id": "ad-atf-", + "gam_unit_path": "/123/news/atf", + "formats": [ + { "width": 300, "height": 250, "media_type": "banner" } + ], + "providers": ["aps"] + }, + "evidence": { + "dom_id": "ad-atf-0", + "gpt": { + "gam_unit_path": "/123/news/atf", + "div_id": "ad-atf-0", + "sizes": [[300, 250]] + } + }, + "warnings": [] + } + ], + "extra_evidence": [], + "warnings": [] + } + ], + "warnings": [] +} +``` + +Warning entries are objects with stable `code` and human-readable `message` +fields. Human output may print only the message. JSON consumers must not need to +parse warning strings. + +Extra live evidence is structured: + +```json +{ + "kind": "gpt", + "phase": "initial_load", + "dom_id": "ad-right-rail-0", + "gam_unit_path": "/123/publisher/right-rail", + "sizes": [[300, 250]], + "reason": "no_configured_slot_matched" +} +``` + +Allowed `kind` values for Phase 1 are `dom`, `gpt`, and `aps`. + +Strict-mode failures with page results use the same shape and set `ok` to +`false`. Example partial slot: + +```json +{ + "ok": false, + "strict": true, + "pages": [ + { + "url": "https://www.example.com/", + "final_url": "https://www.example.com/", + "requested_path": "/", + "path": "/", + "runtime_ad_stack_expected": "unknown", + "gates": { + "method_get": "pass", + "navigation": "pass", + "not_prefetch": "pass", + "not_bot": "pass", + "matched_slots": "pass", + "auction_enabled": "pass", + "consent_allows_auction": "unknown" + }, + "matched_slot_count": 1, + "slots": [ + { + "id": "homepage-header", + "status": "partial", + "phase": "initial_load", + "configured": { + "div_id": "ad-header-0-", + "gam_unit_path": "/123/homepage/header", + "formats": [{ "width": 728, "height": 90, "media_type": "banner" }], + "providers": ["aps"] + }, + "evidence": { + "dom_id": "ad-header-0-_R_abc123", + "gpt": null + }, + "warnings": [ + { + "code": "dom_without_gpt", + "message": "DOM element matched, but no GPT slot evidence was observed" + } + ] + } + ], + "extra_evidence": [], + "warnings": [] + } + ], + "warnings": [] +} +``` + +For errors that occur before any page result can be produced, the command exits +non-zero and prints the normal CLI error. JSON error output can be added later +if the base CLI standardizes it. + +For multi-URL runs, browser/navigation failures after argument validation are +page-level failures when possible. The command continues to the remaining URLs, +sets top-level `ok` to `false`, and includes a page result: + +```json +{ + "url": "https://www.example.com/broken", + "final_url": null, + "requested_path": "/broken", + "path": null, + "error": { + "code": "navigation_failed", + "message": "failed to read main document navigation response" + }, + "slots": [], + "extra_evidence": [], + "warnings": [] +} +``` + +Invalid schemes are still rejected before browser launch for the whole command, +because they are argument errors rather than page collection results. + +## 9. Error Handling + +Static commands fail when: + +- config cannot be loaded; +- `[creative_opportunities]` is malformed; +- CLI assertions in `check` fail. + +Browser verification fails when: + +- config cannot be loaded; +- any URL is not HTTP(S); +- Chrome/Chromium cannot be found or launched; +- all navigations fail before any page result can be collected; +- at least one page-level error occurs in a multi-URL run; +- command output cannot be written; +- `--strict` is set, runtime verification is not skipped by a known gate, and + at least one matched slot is missing or partial. + +Browser collection can still produce a page result with warnings when: + +- page settle times out; +- a navigation redirects before final URL matching; +- scroll evidence is incomplete; +- GPT is not loaded; +- APS is not observed; +- provider evidence is ambiguous; +- extra live DOM/GPT/APS ad-slot evidence has no matched configured slot; +- no slots match the URL. + +## 10. Testing + +Static tests: + +- parse every `ts config ad-templates` command; +- load temp `edgezero.toml` and temp `trusted-server.toml`; +- verify `--app-config`, `--manifest`, and `--no-env` behavior; +- verify `/`, `/news/*`, and full URL normalization behavior; +- verify `check` success and failure output. +- verify the existing static command loader uses the shared `app_config` module. + +Pure comparison tests: + +- exact DOM ID match; +- prefix DOM ID match for framework-generated suffixes; +- ignore `-container` elements; +- GPT confirms by GAM unit path, div ID, and compatible sizes; +- DOM-only creates `partial`; +- no DOM/GPT creates `missing`; +- APS match creates no provider warning; +- APS missing/ambiguous creates provider warnings; +- `--strict` fails only missing and partial slots. + +Browser fixture tests: + +- local HTML fixture with direct `googletag.defineSlot`; +- fixture using `googletag.cmd.push`; +- fixture assigning `window.googletag` after collector injection; +- fixture with delayed/lazy slot observed only with `--scroll`; +- fixture with APS `fetchBids`; +- fixture assigning `window.apstag` after collector injection; +- redirect fixture that matches expected slots on final path; +- multi-URL fixture where one URL fails and one URL returns page results; +- fixture where `[auction].enabled = false` reports runtime skipped instead of + strict missing-slot failure; +- invalid non-HTTP(S) URL rejection before browser launch; +- JSON contract tests for warning codes, `extra_evidence`, page errors, + deterministic ordering, `partial`, `missing`, and strict failures; +- fixture with no real GPT/APS network dependency. + +Verification commands: + +```bash +cargo test --workspace +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test --package trusted-server-cli --target +``` + +## 11. Branch And PR Plan + +The implementation should not be built on stale #724. + +Recommended dependency order: + +1. Land or rebase PR #799 as the CLI base. +2. Rebase PR #800 onto the latest #799 head so `ts audit` uses the current typed + blob app-config model. +3. Harden and refactor the existing static `ts config ad-templates ...` + diagnostics on top of the current server-side ad-template branch and #799: + extract the private config loader into `app_config`, move pure expected-slot + logic into `ad_templates::expected`, and keep existing behavior covered by + tests. +4. Extend the rebased #800 collector with pre-navigation init scripts, + ad-template evidence hooks, optional scroll, page-level errors, and bounded + structured output. +5. Build `ts audit ad-templates verify` on top of that collector and the + server-side ad-template branch. +6. Keep `generate` for a separate Phase 2 spec and PR. + +If delivery needs to be split, static diagnostics can land before browser-backed +verification. Browser-backed verification should not duplicate the #800 browser +collector. + +## 12. CLI Namespace Decision + +`ts audit ad-templates verify` is the final command shape for browser-backed +ad-template verification. + +When this work is combined with the rebased #800 audit command, `ts audit` +should become a subcommand namespace: + +```bash +ts audit page +ts audit generate +ts audit ad-templates verify ... +``` + +The existing #800 `ts audit ` behavior should be preserved as a +compatibility alias for `ts audit generate ` during the transition, +including its artifact output flags. This avoids a successful but silent +behavior change for existing onboarding scripts. + +Parsing contract: + +- `ts audit page ` is the canonical generic page-audit command. +- `ts audit generate ` is the canonical artifact-generation command. +- `ts audit ad-templates verify ...` is the canonical ad-template verifier. +- `ts audit ` is a hidden compatibility alias for + `ts audit generate ` and is accepted only when `` parses as `http` + or `https`. +- `ts audit ad-templates` must never be treated as a legacy URL positional. +- `ts audit page` without a URL must fail with the normal Clap missing-argument + error. + +Implementation shape: + +```rust +#[derive(Debug, clap::Args)] +struct AuditArgs { + #[command(subcommand)] + command: Option, + #[arg(value_parser = parse_http_url, hide = true)] + legacy_url: Option, +} + +#[derive(Debug, clap::Subcommand)] +enum AuditSubcommand { + Page(PageAuditArgs), + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), +} +``` + +If Clap cannot enforce the optional-subcommand plus hidden positional contract +cleanly, implement a small custom dispatcher for the `audit` argv tail and test +it directly. Required parser tests: + +- `ts audit https://www.example.com/` dispatches to artifact generation; +- `ts audit page https://www.example.com/` dispatches to page audit; +- `ts audit ad-templates verify https://www.example.com/` dispatches to + ad-template verification; +- `ts audit ad-templates` does not parse as a URL; +- `ts audit ftp://www.example.com/` fails before browser launch. + +JSON error output is intentionally left to the broader CLI output contract. This +spec only standardizes successful verification result JSON and strict-mode +verification failure JSON where page results exist. diff --git a/docs/superpowers/specs/2026-07-06-cache-control-header-design.md b/docs/superpowers/specs/2026-07-06-cache-control-header-design.md new file mode 100644 index 000000000..1924381f1 --- /dev/null +++ b/docs/superpowers/specs/2026-07-06-cache-control-header-design.md @@ -0,0 +1,145 @@ +# Define Cache-Control strategy for TS edge environment + +**Labels:** `enhancement`, `edge`, `performance`, `caching` +**Area:** Trusted Server runtime (Fetch from Origin / Edge to Clients) + +## Summary + +Trusted Server needs a structured cache policy that separates browser caching from edge/shared caching. Today several TS-owned responses use short, hard-coded cache headers even when the URL is content-versioned. The initial implementation should focus on TS-owned and explicitly fingerprinted assets, especially TSJS bundles. + +Dynamic HTML/template caching, streaming fixes, SSAT compression offload, and Akamai-specific cache behavior are follow-up work and are not part of this cache-header slice. + +## Goals + +- Express cache policy as structured data instead of ad hoc header strings. +- Independently control browser TTL and edge/shared-cache TTL. +- Serve TSJS bundles with immutable caching when the requested URL hash matches the served bytes. +- Keep non-versioned or config-dependent stable URLs out of long-lived immutable browser caches. +- Provide configurable cache-rule presets for known fingerprinted framework paths, starting with Next.js `/_next/static/*`. +- Keep SSAT-assembled ad-stack HTML private and out of shared caches. +- Emit the correct MVP runtime edge-cache headers from the shared policy. + +## Non-goals for this slice + +These are tracked separately: + +- True SSAT publisher streaming and parser-safe assembly: #857. +- SSAT HTML compression offload via `Accept-Encoding: identity` and `X-Compress-Hint`: #858. +- Origin-template caching, transformed-template caching, and dynamic HTML/RSC/API cache-key design: #859. +- Akamai-specific `CDN-Cache-Control` / `Edge-Control` / Property Manager behavior. + +## Background: two cache tiers + +The request path has two cacheable hops: + +```text +Origin ──▶ TS edge/shared cache ──▶ Browser cache +``` + +- **Edge/shared cache:** controlled by `s-maxage` or runtime-specific edge headers. + - Fastly: `Surrogate-Control` + - Cloudflare: `CDN-Cache-Control` / `Cloudflare-CDN-Cache-Control` + - Portable fallback: `s-maxage` inside `Cache-Control` +- **Browser cache:** controlled by `max-age` and related `Cache-Control` directives. + +A single `max-age` cannot express “hold at the edge for a year, but revalidate in the browser daily” or the reverse. TS should model these tiers separately and let adapters render the appropriate headers. Header rendering alone does not enable a runtime cache: Cloudflare Workers Cache must be enabled, and Fastly synthetic/final egress responses require explicit cache integration tracked in [#908](https://github.com/IABTechLab/trusted-server/issues/908). + +## Policy model + +Add a shared cache-policy model along these lines: + +```text +match +edge_ttl +browser_ttl +stale_while_revalidate +stale_if_error +immutable +visibility: public | private +enabled +``` + +Rules should be configurable. Built-in framework presets, such as Next.js `/_next/static/*`, should be represented as default rules in this same model rather than hard-coded in adapters. Operators must be able to disable or override presets and add publisher-specific allowlists. + +## Target behavior by response class + +| Response class | Target policy | Notes | +| --------------------------------------------------------------- | ----------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| TSJS with matching `?v=` | `Cache-Control: public, max-age=31536000, immutable` plus runtime edge header | The serving path must validate that `v` matches the bytes served. | +| TSJS missing/mismatched `?v=` | Short TTL or redirect to canonical hashed URL | Do not mark immutable. | +| TSJS deferred modules, including Prebid | Same as TSJS hash-matching policy | Example: `/static/tsjs=tsjs-prebid.min.js?v=`. | +| Publisher Prebid URL neutralized by TS | `no-store` or very short TTL | The empty compatibility shim is config-dependent and served at a stable publisher URL. Do not cache it for a year. | +| Enabled framework preset static, e.g. Next.js `/_next/static/*` | `Cache-Control: public, max-age=31536000, immutable` | Applied through configurable preset/allowlist rules. | +| Fastly TS-fingerprinted rehosted asset | `Cache-Control: public, max-age=31536000, immutable` | Safe only when TS owns the fingerprinted URL. A matched rehost rule is authoritative over third-party origin cache defaults. | +| Stable Fastly TS-owned/rehosted URL | Conservative short browser TTL, optional longer edge TTL | Do not use `immutable`; later TS/operator `private` or `no-store` finalization remains a veto. | +| Arbitrary publisher-origin CSS/JS/images | Origin-controlled by default | TS may upgrade only via enabled framework preset or publisher allowlist. | +| SSAT-assembled ad-stack HTML | `Cache-Control: private, max-age=0`; strip runtime edge-cache headers | Must never enter shared cache because it can contain per-user slot/bid data. | +| Dynamic HTML/RSC/API | Origin-controlled in this slice | Future dynamic caching belongs to #859. | + +## TSJS-specific requirements + +Current TSJS URLs already include a content hash query string, for example: + +```text +/static/tsjs=tsjs-unified.min.js?v= +/static/tsjs=tsjs-prebid.min.js?v= +``` + +The current serving path still emits a short cache policy. Update it so that: + +- hash-matching requests emit one-year immutable browser caching; +- hash-matching requests emit the runtime edge header with equivalent long edge TTL; +- missing or mismatched hash requests do not receive immutable caching; +- cache-key configuration preserves the `v` query parameter; +- TSJS hashes used in injected URLs are generated at build time or cached so HTML injection does not re-concatenate and re-hash large bundles per pageview; +- `Vary: Accept-Encoding` remains on compressed/static responses; +- ETags may remain as a fallback for clients or intermediaries that revalidate anyway. + +Fastly and Cloudflare include query strings in default cache keys, but TS must still avoid any project-specific query normalization that drops `v` for `/static/tsjs=`. + +## SSAT HTML privacy requirement + +SSAT-assembled ad-stack HTML can contain per-user data such as slot state or bid data. It must remain: + +```http +Cache-Control: private, max-age=0 +``` + +and must strip runtime edge-cache headers, including: + +```http +Surrogate-Control +Fastly-Surrogate-Control +CDN-Cache-Control +Cloudflare-CDN-Cache-Control +``` + +This requirement applies to the browser-facing assembled response. Origin-template caching is separate follow-up work in #859. + +## Runtime header mapping for MVP + +Adapters should render the shared policy as follows: + +| Runtime | Edge/shared-cache header | +| ----------------- | ---------------------------------------------------- | +| Fastly | `Surrogate-Control` | +| Cloudflare | `CDN-Cache-Control` / `Cloudflare-CDN-Cache-Control` | +| Portable fallback | `s-maxage` in `Cache-Control` | + +These mappings define emitted directives, not storage by themselves. The runtime must enable or implement the corresponding shared-cache mechanism. Akamai mapping is deferred until Akamai is on the roadmap. + +## Acceptance criteria + +- [x] Cache policy is represented as structured fields, not hard-coded header strings. +- [x] Built-in framework presets, including a disableable/overrideable Next.js `/_next/static/*` rule, are implemented through the shared cache-policy rule engine. +- [x] TSJS hash-matching requests for unified and deferred modules emit `public, max-age=31536000, immutable` plus the runtime edge header. +- [x] TSJS missing/mismatched hash requests do not get immutable caching. +- [x] TSJS hash generation is build-time or cached enough that HTML injection does not re-concatenate/re-hash the bundle per pageview. +- [ ] Runtime cache-key configuration preserves the `v` query parameter for `/static/tsjs=`. Runtime verification is tracked in #908. +- [x] Neutralized publisher Prebid shim responses use `no-store` or a short TTL, not a year-long policy. +- [x] Arbitrary publisher-origin assets remain origin-controlled unless covered by an enabled framework preset or publisher allowlist. +- [x] Fastly TS-owned rehosted assets have explicit normalized cache policy; immutable is used only for TS-fingerprinted rehosted URLs. +- [x] SSAT-assembled ad-stack HTML continues to emit `private, max-age=0` and strips all runtime edge-cache headers. +- [x] Fastly and Cloudflare adapters emit the correct edge-cache header from the shared policy, with portable `s-maxage` fallback where needed. Actual shared-cache storage remains tracked in #908. +- [x] Dynamic HTML/RSC/API Vary/cache-key normalization is not hard-coded in this PR and is deferred to #859. +- [x] SSAT streaming fixes and compression offload are not included in this PR and remain tracked by #857 and #858. diff --git a/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md b/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md new file mode 100644 index 000000000..38ffa7ef4 --- /dev/null +++ b/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md @@ -0,0 +1,1141 @@ +# GAM `ts=true` attribution for Trusted Server A/B traffic + +Date: 2026-07-15 + +Updated: 2026-08-14 + +Status: Design + +## Problem + +A publisher will route a small, cookie-sticky A/B cohort through Trusted Server +while the control cohort continues through the existing production path. The +publisher wants Google Ad Manager (GAM) reporting to identify impressions and +clicks generated on pages served through Trusted Server and compare them with +the unmodified production cohort. + +Trusted Server currently adds the slot-level key-value `ts_initial=1` while it +prepares initial GPT slots. That key has a different lifecycle and meaning from +the experiment marker: + +- it identifies the initial slot request prepared by Trusted Server; +- it is cleared before later client-side refresh auctions; and +- it is set on matched slots rather than every GPT request on the page. + +The experiment needs a document-delivery marker. When attribution is explicitly +enabled, every request issued by the document-local GPT PubAds service after +Trusted Server rewrites and emits the document head must contain `ts=true`, +including initial requests, publisher-owned slots, lazy slots, and refreshes. +Production documents cannot be modified, so the control cohort remains +unmarked. + +## Goals + +1. Add `ts=true` to every in-scope GPT PubAds request made after Trusted Server + rewrites and emits the document head. +2. Leave production/control pages unchanged. +3. Preserve the existing `ts_initial=1` slot-ownership and refresh lifecycle. +4. Support GAM reports that count treatment impressions and clicks and derive + the control counts within the same experiment scope. +5. Avoid changing auction eligibility, ad delivery, consent behavior, or page + performance. +6. Define the data-quality checks needed when an unmarked request is used as the + control baseline. +7. Keep attribution disabled by default and independently reversible without + disabling the GPT integration. + +## Non-goals + +- Implement or change the cookie-based A/B router. The experiment infrastructure + owns sticky cohort assignment and routes only the treatment cohort through + Trusted Server. +- Prove that a Trusted Server server-side bid won the GAM auction. For an + in-scope document satisfying the non-cloned activation prerequisite, `ts=true` + means that Trusted Server emitted the document head through its publisher + pipeline, regardless of whether the winning demand was a server-side bid, a + direct GAM line item, Ad Exchange, or backfill. +- Mark GAM traffic outside a document-local GPT PubAds service whose head was + processed by the enabled publisher attribution pipeline. IMA/video SDK + requests, direct tags, and server-side GAM requests require separate + instrumentation and are outside this design. Nested inventory is eligible + only when its own response is independently routed, rewritten, and validated; + the activation attribute is not proof of those facts because publisher code + can copy it into `srcdoc` or `document.write` markup. +- Replace `ts_initial`, `hb_*`, line-item, bidder, or creative reporting. +- Add a client-side analytics beacon or a Trusted Server telemetry event. +- Make GAM click tracking more complete. The marker only segments clicks that + GAM already records. +- Provide billing-grade or causal experiment analysis from GAM alone. + +## Assumptions and prerequisites + +- The GPT integration and its separate `gam_attribution_enabled` setting are + enabled on every Trusted Server deployment receiving treatment traffic. The + attribution setting defaults to `false`; enabling GPT alone does not emit the + marker or its activation signals. +- The response enters Trusted Server HTML rewriting, contains a literal `` + element, and Trusted Server rewrites and emits that head before any publisher + script issues a GAM request. Pass-through or buffered-unmodified responses and + origin markup that omits `` cannot satisfy the marker guarantee. +- On the existing no-post-processor Fastly streaming path, the rewritten head + can reach the browser before origin EOF or rewriter finalization. `ts=true` + therefore certifies successful head rewrite and emission, not successful + completion of the remaining body. A later origin, decode, or rewrite failure + can truncate an already marked page; the request remains treatment traffic and + the delivery failure is monitored separately. This design adds no response + buffering; configurations with HTML post-processors retain their existing + buffering behavior. +- The publisher Content Security Policy allows Trusted Server's bare inline + scripts to execute. Initial `adSlots`, the GPT enable flag, the GPT bootstrap, + and the `bids`/`adInit` invocation are all nonce-less inline scripts; Trusted + Server does not currently propagate a publisher nonce or update CSP hashes. A + policy that blocks those scripts makes the initial TS ad stack inert and is + ineligible at launch even if it allows the synchronous first-party TSJS + bundle. +- Each in-scope HTML document uses one document-local GPT PubAds service. A + marked parent does not mark a nested GPT instance. The publisher-only bundle + attribute is deliberately non-secret and can be copied, so the implementation + cannot guarantee that an unrewritten nested document never activates the + fallback. IMA/video, direct-tag, server-side GAM, and nested inventory not + independently routed, rewritten, and validated must be excluded from the + experiment and paired reports. A copied activation attribute is a measurement + contamination incident, not evidence that the nested document is eligible. +- Treatment and control traffic use the same GAM network and comparable + inventory. Report filters can isolate the pages and time window eligible for + the experiment. +- The experiment owner can obtain the expected treatment allocation from the + cookie router, even though Trusted Server does not read or emit that cookie. +- Publisher code and Trusted Server creative-opportunity slot configuration do + not reuse `ts` for another meaning, set a slot-level `ts` value, or clear + page-level targeting after Trusted Server targeting runs. The deployment audit + must inspect `trusted-server.toml` targeting maps and search publisher code + for `setTargeting`, `setConfig`, and `clearTargeting` uses that could + overwrite or remove the reserved key. If such behavior exists, it must be + resolved before launch; silently filtering operator targeting or wrapping + publisher GPT APIs is out of scope. + +## Existing behavior + +The GPT integration has two related pieces: + +1. `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` is injected at + the start of ``. It creates the GPT command queue early and installs + the minimal `window.tsjs.adInit` implementation used before the richer bundle + is available. +2. `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` installs the + richer GPT integration and applies slot-level auction targeting. + +Both initial-render paths set `ts_initial=1` on slots handled by `adInit`. The +Prebid refresh integration includes `ts_initial` in its list of stale +slot-targeting keys and clears it before subsequent client-side refresh +auctions. SPA cleanup also clears stale `ts_initial` targeting before applying +new route state. + +This behavior is correct for `ts_initial` and must not change. It is not +sufficient for a page-level treatment marker because it does not cover all GPT +slots and intentionally does not persist across refreshes. + +## Decision + +Add a separate page-level GPT key-value: + +```text +ts=true +``` + +The marker contract is fixed: the target GAM network rejected the longer +`trusted_server` request name under its enforced 10-character limit. Trusted +Server therefore emits exactly `ts=true`, with no configurable key or value and +no alias or dual-write path. + +Attribution has a separate, disabled-by-default GPT setting: + +```toml +[integrations.gpt] +gam_attribution_enabled = false +``` + +The equivalent environment override is +`TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED`. Trusted Server's +environment overlay only replaces leaves that are present in the TOML source, +so an operator relying on that override must retain +`gam_attribution_enabled = false` under `[integrations.gpt]` in the base file. +When the setting is `false`, enabling GPT preserves current behavior: the dormant +gated bootstrap code may still be present, but no attribution callback is +enqueued or executed and no inline attribution flag, bundle activation +attribute, or fallback is activated. This setting is the attribution kill switch +and does not disable GPT proxying, script rewriting, the GPT shim, `adInit`, or +`ts_initial`. + +When attribution is enabled, `head_inserts` adds +`window.__tsjs_gam_attribution_enabled=true` to the integration's existing first +inline head insert; it does not add a third insert. That inline flag authorizes +the raw bootstrap's primary marker path. It is deliberately not the bundle +fallback signal because CSP can block the inline insert that defines it. + +The early GPT bootstrap will enqueue page-level targeting before publisher GPT +commands execute only when the inline attribution flag is exactly `true`. The +enqueue must occur after `window.tsjs` is initialized but before the existing +`if (ts.adInit) return;` guard: + +```text +(function () { + if (typeof window === "undefined") return; + var ts = (window.tsjs = window.tsjs || {}); + var tag; + if (window.__tsjs_gam_attribution_enabled === true) { + tag = (window.googletag = window.googletag || { cmd: [] }); + tag.cmd = tag.cmd || []; + tag.cmd.push(function () { + try { + if (typeof googletag.setConfig === "function") { + googletag.setConfig({ targeting: { ts: "true" } }); + } + } catch (_) { + // Attribution must not interrupt the existing bootstrap. + } + }); + } + + if (ts.adInit) return; + // Existing initial-load detector and adInit stub follow, reusing `tag` when + // attribution initialized it and preserving their current path otherwise. +})(); +``` + +The exact implementation must follow the repository's JavaScript formatting and +defensive checks. The important contract is that the page-level targeting +command is queued by the head bootstrap before the origin page can queue its GPT +setup or request ads when attribution is enabled. Page attribution is +independent of whether the bootstrap needs to install `ts.adInit`, so the +existing guard may skip only the ad-init stub and detector setup, never an +enabled marker enqueue. An unavailable targeting API may skip only the marker +callback; it must not prevent later queued publisher or Trusted Server callbacks +from running. + +The existing initial-load detector immediately below the new marker reuses the +initialized `tag.cmd` reference when attribution created it and follows its +current initialization path otherwise. In particular, attribution disabled plus +a pre-existing `ts.adInit` must still return without creating +`window.googletag`; the new default-off setting cannot change current runtime +behavior. + +Moving queue initialization above the `ts.adInit` guard intentionally creates a +standard `window.googletag` command-queue stub on every attribution-enabled page, +including a page where `ts.adInit` already exists and GPT never loads. The stub +is inert by itself and preserves the marker-before-guard guarantee; it is not an +accidental behavior to remove during implementation review. + +The TypeScript GPT bundle will defensively enqueue the same page-level targeting +at module initialization after the existing flag-gated shim block and before +`installTsAdInit()`, only when the executing publisher-page bundle tag carries +the non-executable `data-ts-gam-attribution="true"` attribute. The HTML pipeline +adds that attribute only when GPT and `gam_attribution_enabled` are both enabled. +A pre-existing `ts.adInit` is already covered by placing the bootstrap marker +before the guard and is not a reason for the fallback. + +The fallback exists to preserve delivery-path attribution if the inline +bootstrap unexpectedly stops executing while the synchronous first-party bundle +still runs before publisher GPT. It does not recover `adSlots`, bids, the +`adInit` invocation, or the initial TS auction: those are also nonce-less inline +scripts. On an eligible publisher document whose activation tag was not copied, +a fallback-only marker therefore still truthfully means "document head emitted +through Trusted Server," but it also indicates a deployment state that was +ineligible at launch. Synthetic validation must treat that state as a +measurement incident, pause interpretation, and exclude the affected time window +from both paired reports if the incident contaminates collected results. GAM +cannot distinguish fallback-only pages from normally executing treatment pages +because both intentionally use the same marker. + +Neither targeting path can cover a response that was not HTML-rewritten, markup +without ``, a policy that blocks both injected paths, or a publisher GPT +request issued before the injected head content runs. Those are deployment +eligibility and coverage-validation concerns, not runtime conditions the +targeting code can repair. + +The implementation uses GPT's current page-level `googletag.setConfig` API +rather than the deprecated `pubads().setTargeting()` API. See +[GPT configuration API migration](https://developers.google.com/publisher-tag/guides/config-migration). +Page-level targeting is the right scope because GPT applies it to all slots +associated with the `pubads` service. Once installed, it remains effective for +initial, lazy, and refreshed requests for the life of the page. Existing slot +targeting may add or override other keys without requiring Trusted Server to +discover every publisher slot. + +GPT merges page-level targeting per key across `setConfig` calls. Enqueuing +`ts=true` from both Trusted Server paths is therefore idempotent, and a +publisher call that sets an unrelated targeting key preserves `ts`. The explicit +clear operations are a per-key `null`, a whole-targeting `null`, or the +equivalent legacy `pubads().clearTargeting()` calls. See +[GPT key-value targeting](https://developers.google.com/publisher-tag/guides/key-value-targeting). + +`ts` is intentionally not added to the slot-targeting cleanup arrays. Those +arrays manage per-auction state. Clearing page-level `ts` during refresh or SPA +navigation would incorrectly move a treatment page into the unmarked control +cohort. + +## Attribution contract + +### Treatment + +An in-scope GAM request is in the treatment cohort when it contains: + +```text +ts=true +``` + +For an in-scope document that satisfies the non-cloned activation prerequisite, +the marker means: + +> Trusted Server rewrote and emitted the containing document's `` through +> the enabled publisher attribution pipeline before the in-scope GPT request. + +It does not mean: + +- the complete streamed response reached the browser or finished rewriting; +- a Trusted Server bidder returned a bid; +- a Trusted Server bid won; +- Trusted Server rendered the winning creative; or +- the request was the first impression for the slot. + +### Control + +The production path cannot be changed. Within the exact experiment inventory, +time window, and publisher scope, an unmarked GAM request is treated as control. + +This is an inference rather than an explicit `ts=false` assertion. A treatment +request that loses its marker would be misclassified as control. The rollout +therefore requires coverage checks that compare the observed GAM treatment share +with the A/B router's expected cookie cohort share. + +### Relationship to `ts_initial` + +| Key | Scope | Lifetime | Meaning | +| -------------- | ---------- | ---------------------------- | --------------------------------------------------- | +| `ts=true` | Page-level | Entire browser page lifetime | Eligible, non-cloned TS pipeline emitted page head | +| `ts_initial=1` | Slot-level | Initial TS-managed request | Initial slot request was prepared by Trusted Server | + +The two keys answer different questions and coexist. No code or report should +infer that one is an alias for the other. The value contract is exactly +`ts=true`: do not emit, accept, or report any other value (for example `ts=1`), +and do not dual-write an alternative key name such as `trusted_server`. + +## Request lifecycle + +```text +Sticky A/B cookie + -> control: browser receives production page + -> publisher GPT runs without the `ts` key + -> treatment: request is routed through Trusted Server + -> deployment has `gam_attribution_enabled = true` + -> GPT head bootstrap queues page-level `ts=true` + -> GPT bundle defensively queues the same marker + -> GPT library drains the command queue + -> publisher and TS define/display/refresh slots + -> every in-scope PubAds request carries `ts=true` +``` + +The marker covers: + +- Trusted Server-defined initial slots; +- publisher-defined slots reused by Trusted Server; +- publisher slots that are not part of a Trusted Server creative opportunity; +- slots created lazily after initial page load; +- publisher-initiated refreshes; +- Prebid-managed refreshes; and +- SPA route changes within the same browser document. + +All bullets refer to slots using the same document-local GPT PubAds service. +Requests from IMA/video SDKs, direct tags, or server-side GAM integrations are +not covered merely because the containing document is marked. Nested inventory +is eligible only when Trusted Server separately routes and rewrites that +document and validation proves the same request and report contract. Because a +publisher can copy the non-secret activation tag into `srcdoc` or +`document.write` content, marker presence alone does not prove that a nested +document was independently rewritten. + +A full browser navigation creates a new page and repeats cookie-based routing. +The new page receives the marker only when that navigation is routed through a +Trusted Server deployment with `gam_attribution_enabled = true`, its head is +rewritten and emitted, and a marker callback successfully applies page-level +targeting before its first in-scope GPT request. + +## Component changes + +### GPT configuration and head activation + +`crates/trusted-server-core/src/integrations/gpt.rs` will add +`gam_attribution_enabled: bool` to `GptConfig` with Serde's ordinary `false` +default. Configuration tests must cover an omitted field, explicit `false`, +explicit `true`, and an environment override whose base TOML includes the leaf. +The example configuration will show the field as disabled. + +`GptIntegration::head_inserts` keeps its current number and order of scripts. +When attribution is enabled, it appends the inline attribution flag to the +existing GPT enable/shim insert; when disabled, that insert remains byte-for-byte +equivalent to its current behavior apart from formatting that does not affect +execution. + +The same parsed `GptConfig` instance must authorize the publisher bundle-tag +attribute without reparsing settings or relying on head-insert side effects. +Add a default-empty `IntegrationHeadInjector::tsjs_script_tag_attributes()` +hook, override it in `GptIntegration` to return only +`data-ts-gam-attribution="true"` when attribution is enabled, and aggregate the +attributes through `IntegrationRegistry`. `html_processor.rs` passes that +registry-owned attribute list to a new publisher-only +`tsjs_script_tag_with_attributes` helper. Keep the existing +`tsjs_script_tag` and `tsjs_unified_script_tag` output unchanged for creative, +test, and other generic callers. This makes the bootstrap and fallback derive +from one integration-owned setting while avoiding a GPT-specific +`HtmlProcessorConfig` field or order-dependent document-state mutation. + +### Early GPT bootstrap + +`crates/trusted-server-core/src/integrations/gpt_bootstrap.js` owns the +behavior. It will set the page-level key in its earliest GPT command callback, +before the `ts.adInit` early-return guard, only when the inline attribution flag +is exactly `true`. The targeting code stays inside the existing raw bootstrap +script returned by `head_inserts`; it must not add a third head insert. + +The operation must be idempotent. Calling +`googletag.setConfig({ targeting: { ts: 'true' } })` more than once with the +same value is harmless, but the bootstrap should avoid adding a new global state +machine solely for deduplication. + +The bootstrap already binds the local variable `ts` to the `window.tsjs` +namespace, so the targeting key `ts` and that variable are unrelated names that +sit only a few lines apart. Add a clarifying comment at the targeting call so a +maintainer does not read the key as the namespace. The value is the string +`'true'`, never the boolean `true`: GPT targeting values must be strings. + +### TypeScript GPT bundle fallback + +`crates/trusted-server-js/lib/src/integrations/gpt/index.ts` will add a small +`installTrustedServerPageTargeting()` helper and call it during GPT module +initialization after the existing flag-gated `installGptShim()` block and before +`installTsAdInit()` when the publisher-page bundle's activation attribute is +present. The helper creates or reuses the standard GPT command queue, enqueues +the same defensive `setConfig({ targeting: { ts: 'true' } })` call, and does not +read the experiment cookie or wait for an auction. The existing optional +`GoogleTag.setConfig` method and `GoogleTagConfig extends Record` types already cover this call and must be reused rather than extended. + +The bootstrap remains the primary path because it is injected first. The bundle +call is a redundant fallback and must not delay module initialization, create a +request, or add slot-level targeting. The attribution helper must not create +`window.googletag` independently when the activation attribute is absent. A +standalone module import with neither the existing GPT-enable flag nor the new +attribute must preserve the current runtime-gating contract and leave +`window.googletag` untouched. + +### Non-executable bundle activation + +The publisher HTML pipeline in +`crates/trusted-server-core/src/html_processor.rs`, using a separate, +publisher-page-only tag helper in `crates/trusted-server-core/src/tsjs.rs`, will +add a `data-ts-gam-attribution="true"` attribute to the existing synchronous +`#trustedserver-js` bundle tag only when GPT and `gam_attribution_enabled` are +both enabled. The attribute is data, not an inline executable, so CSP can block +the inline GPT head inserts while still allowing the external bundle to detect +that it owns page attribution. + +At module initialization, the GPT bundle captures `document.currentScript` and +requires that executing synchronous script to carry +`data-ts-gam-attribution="true"` before the fallback may create a GPT stub. It +must fail closed when the executing script cannot be identified. Do not +authorize activation through a global `#trustedserver-js` lookup: the generic +unified tag uses the same ID in creative and test contexts, and duplicate IDs +could select the wrong element. Binding the signal to the executing tag keeps +the activation decision explicit and testable without relying on an inline +global flag. + +The existing `window.__tsjs_gpt_enabled` flag continues to activate +`installGptShim()` when inline scripts run. It cannot activate the CSP fallback +because the server sets it from an inline head insert—the execution path CSP may +block. Migrating shim activation to the data attribute is out of scope; module +initialization preserves the current flag-gated shim installation, then runs the +attribute-gated page-targeting helper, then installs `ts.adInit` and the +remaining GPT bundle hooks. + +This signal must be limited to the publisher-page bundle generated from the +enabled integration registry. Do not infer activation merely because the GPT +module exists in an all-modules bundle: creative and test tooling can load that +bundle outside the publisher GPT integration. Do not add a new script tag or +change the integration's existing head-insert count. + +Extend the `tsjs.rs` and `html_processor.rs` tests to prove that the existing +publisher-page bundle tag gains the activation attribute only when GPT and +`gam_attribution_enabled` are both enabled, remains a single external tag, and +omits the attribute for attribution-disabled, non-GPT, and generic all-modules +bundles. Bundle tests must also prove that an unrelated or duplicate element +with `id="trustedserver-js"` cannot activate the fallback. + +### GPT Rust integration tests + +`crates/trusted-server-core/src/integrations/gpt.rs` already tests the embedded +bootstrap returned by `head_inserts`. Extend those tests to prove that: + +- omitted and explicit-false configuration emit neither the inline attribution + flag nor publisher-tag attribute metadata; +- explicit-true configuration emits the inline attribution flag and exposes the + publisher-tag attribute metadata; +- the bootstrap contains the flag-gated page-level `ts=true` targeting; +- the marker enqueue appears before the `if (ts.adInit) return;` guard; +- the targeting setup is queued before `ts.adInit` can issue `display` or + `refresh`; +- the existing `ts_initial` marker remains present; and +- the attribution-enabled integration without `slim_prebid_url` still emits + exactly the + existing two head inserts, proving the marker was added to the bootstrap + instead of a new tag. + +### Bootstrap execution tests + +Extend the existing +`crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts` +Vitest/jsdom behavioral test for the raw bootstrap. Preserve its established +`process.cwd()` source resolution and `new Function(BOOTSTRAP_SOURCE)` execution +strategy unless a separate test-harness change demonstrates a concrete need to +replace them. The tests continue to execute the checked-in raw source rather +than a copied fixture and add no JavaScript runtime dependency to Rust. + +The harness must prove that: + +- attribution disabled preserves the existing pre-installed-`ts.adInit` behavior + without creating `window.googletag`; +- attribution enabled queues the callback before a publisher callback added + after the injected bootstrap; +- draining the queue calls `googletag.setConfig` with page-level `ts=true` + before the publisher callback runs; +- a pre-existing `ts.adInit` does not prevent the attribution callback from + being queued or executed; +- a publisher callback queued after the bootstrap still runs when + `googletag.setConfig` throws; +- an unavailable or throwing `googletag.setConfig` does not prevent the existing + `disableInitialLoad` wrapper from being installed; +- `ts.adInit` remains installed when attribution setup is unavailable or throws; + and +- calling the wrapped `disableInitialLoad` still records + `ts.gptInitialLoadDisabled`. + +### Bundle fallback tests + +Extend `crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts` using +its existing dynamic-import and `vi.resetModules()` pattern. Prove that module +initialization with the bundle activation attribute queues page-level `ts=true` +after any existing flag-gated shim installation and before installing +`ts.adInit`, that it reuses an existing GPT command queue, and that unavailable +or throwing `setConfig` does not stop the remaining GPT module installers. The +attribution helper must not create a stub independently when its attribute is +absent: a standalone import with neither the existing GPT-enable flag nor the +new attribute leaves `window.googletag` untouched. An attribution-disabled but +GPT-enabled deployment preserves today's flag-gated shim and stub behavior; it +only omits the attribution callback and fallback. A duplicate call after the +bootstrap must remain safe and must not create another script or network +request. + +### Slot cleanup constraints + +No refresh-lifecycle change is required. In particular: + +- do not add `ts` to `TS_REFRESH_TARGETING_KEYS`; +- do not add `ts` to `TS_BASE_TARGETING_KEYS`; +- do not rename or remove `TS_INITIAL_TARGETING_KEY`; and +- do not copy `ts` onto individual slots. + +Leaving these components unchanged is part of the design: slot cleanup cannot +remove a page-level key set through `googletag.setConfig`. + +No new `CreativeOpportunitySlot` parsing, filtering, or startup validation is +added. Trusted Server continues accepting and forwarding operator targeting +maps verbatim. The external launch audit—not runtime code—must reject an +effective configuration containing slot-level `ts`. + +### Documentation + +Document the distinction between page-level `ts=true` and slot-level +`ts_initial=1` in `docs/guide/integrations/gpt.md`, near the existing command +queue documentation. Document `gam_attribution_enabled`, its default-off and +kill-switch behavior, the fixed marker contract, and the GAM setup and reporting +preconditions below. Add the disabled field to `trusted-server.example.toml`; do +not add this current integration to a planned-future GAM document. + +## GAM configuration + +GAM configuration is a deployment prerequisite and must be completed before the +experiment starts because key-value reporting is not retroactive. + +The request contract is finalized as key name `ts`, predefined value `true`. +The target-network preflight rejected `trusted_server` (14 characters) under +the enforced 10-character request-name limit, consistent with the SOAP/REST +`CustomTargetingKey` contract. Because `ts` is short, it carries a real collision +risk with common publisher timestamp or cache-buster keys, which makes the +cross-system collision audit a hard launch gate, not a formality. If any +publisher, Trusted Server configuration, Prebid targeting source, or GAM object +already uses `ts`, the experiment must stop until the collision is removed. +Changing the finalized contract or silently filtering established targeting is +out of scope. + +1. In **Inventory > Key-values**, create or verify the `ts` key and retain the + target-network preflight result with the experiment runbook. The SOAP/REST + [`CustomTargetingKey`](https://developers.google.com/ad-manager/api/reference/v202605/CustomTargetingService.CustomTargetingKey) + documents the enforced 10-character request-name limit and a 40-character + value limit. +2. Use a predefined value named `true`. +3. Preflight whether the target network has Enhanced key-value reporting and + confirm the publisher has approved its Premium reporting activation and + displayed CPM terms, minimum/maximum charge, and non-prorated monthly billing. + Record the result. If approved and compatible with the selected metrics, + enable `ts` as a dedicated Enhanced dimension. Otherwise, enable `ts` for + legacy reporting and prove with a target-network dry run that the exact + legacy `ts=true` filter and chosen metrics are available; never sum unfiltered + legacy **Key-values** rows. See + [Access Premium reporting](https://support.google.com/admanager/answer/16176700) + and [Report on targeting keys](https://support.google.com/admanager/answer/14528835). +4. Reserve `ts` for Trusted Server page attribution. +5. Audit existing publisher GPT code and every GAM object that consumes custom + targeting for an existing `ts` key before deployment. This includes line + items, proposal line items, rules, protections, yield configuration, and any + network-specific custom-targeting surface. +6. Audit Prebid-generated GPT targeting, including `pbjs.bidderSettings`, each + bidder's and standard key's `adserverTargeting`, and every call to + `setTargetingForGPTAsync`. Record every path capable of producing a slot-level + `ts`; any occurrence is a launch blocker because GPT slot targeting overrides + the page-level value. +7. Audit every `CreativeOpportunitySlot.targeting` map from all effective + `trusted-server.toml` configuration sources. The arbitrary operator-supplied + map is copied to GPT slots, where a slot-level `ts` value would override the + page-level marker. Any occurrence is a launch blocker; do not silently + discard it because that could change established operator targeting. +8. Audit publisher code for every operation that can remove or supersede the + marker after initial GPT setup. Search for `setConfig({ targeting: null })`, + a `ts: null` or different `ts` value, `pubads().clearTargeting()` with no key + or with `ts`, and slot-level `ts` targeting. Account for equivalent calls + assembled dynamically. + +The runbook records the audit owner, exact queries or search procedures, result +artifact, timestamp, and re-audit trigger. A deployment, publisher GPT, Prebid, +or GAM targeting change affecting the audited surfaces invalidates the prior +result and blocks attribution until the audit is repeated. + +The audit is a hard precondition. If `ts` already has another meaning, or any +GAM object targets or acts on `ts=true`, the experiment owner must resolve the +collision before deployment. The measurement marker is not intended to change ad +eligibility, pricing, protection, or routing. A pre-existing targeting consumer +for `ts=true` would make the A/B test measure a traffic or demand change at the +same time as Trusted Server delivery. + +Undefined values do not appear in standard key-value reports even when the key +is reportable, so value `true` must exist before treatment traffic begins. See +[Add key-values](https://support.google.com/admanager/answer/9796369) and +[Report on targeting keys](https://support.google.com/admanager/answer/14528835). + +## Reporting and comparison + +### Report scope + +Every comparison must apply identical filters for: + +- publisher/network; +- experiment start and end time; +- sites or inventory included in the cookie experiment; +- ad units and formats; +- geography and device categories, when used; and +- any consent or traffic-quality exclusions. + +Before launch, the experiment owner freezes an authoritative scope manifest +containing those values, the eligible route/site inventory, exact ad-unit list, +owner, version, and activation timestamp. Report A, Report B, router/access-log +queries, synthetic URLs, and any external denominators must all reference that +same manifest. A scope change closes the current reporting window and requires a +new manifest version and fresh preflight. + +Do not compare the TS cohort with all unmarked network traffic unless all that +traffic is eligible for the same experiment. Likewise, exclude smoke tests, +direct hits, operations traffic, and any other TS-served page outside the cookie +experiment. The marker identifies the delivery path, not the router's cohort +assignment, so all such requests also carry `ts=true` when attribution is +enabled for their Trusted Server deployment. + +The route owner must use router or access logs to prove that non-experiment TS +traffic is absent from the eligible inventory during the measurement window. If +such traffic cannot be prevented and has no independent inventory or reportable +dimension, GAM cannot remove it from Report B because its marker is identical to +the cohort marker; the experiment must not launch. Record the owner, query, +expected zero threshold, and response procedure in the experiment runbook. + +Before treatment routing begins, run and retain a zero-count query for every +excluded path: non-experiment Trusted Server traffic, smoke/direct/operations +traffic, IMA/video, direct tags, server-side GAM, and excluded nested inventory. +Any non-zero result blocks launch unless an independent dimension excludes the +same traffic from both saved reports. + +### Cohort calculations + +Create and retain two saved reports with identical date boundaries, time zone, +inventory filters, traffic-quality filters, and metric definitions: + +1. **Report A — experiment total.** Do not include **Placement**, legacy + **Key-values**, **Targeting**, **Yield group**, or another dimension that can + represent one event more than once. This report provides one non-duplicated + total for every metric in the eligible experiment scope. +2. **Report B — TS treatment.** Use the dedicated Enhanced `ts` dimension + filtered to `ts=true`. If Enhanced key-value dimensions are unavailable, + unapproved, or incompatible with the saved metric pair, use the legacy + **Key-values** dimension filtered to exactly `ts=true` and do not sum any + other key-value rows. Do not add **Placement**, **Targeting**, **Yield + group**, or any unrelated dimension that can represent the filtered treatment + event more than once. + +The legacy **Key-values** dimension can emit the same impression or click on +multiple rows when a request contains multiple key-values. It therefore cannot +provide Report A or a summable totals row. See +[Avoid double counting report totals](https://support.google.com/admanager/answer/7642799). + +For this paired report scope, define: + +```text +total_impressions = Report A impressions +ts_impressions = Report B impressions +prod_impressions = total_impressions - ts_impressions + +total_clicks = Report A GAM-recorded clicks +ts_clicks = Report B GAM-recorded clicks +prod_clicks = total_clicks - ts_clicks +``` + +If the selected GAM report exposes an explicit unassigned or `(not set)` row, +that row may be used only as a cross-check. The paired Report A minus Report B +calculation remains the control definition because production cannot send an +explicit value. The experiment owner must retain both report definitions with +the results so later analysis can verify that their filters and metrics match. +Export both reports after the same GAM reporting-latency and invalid-traffic +adjustment window. If GAM restates one report, rerun the pair before applying +the subtraction. + +For every paired metric and reporting window, validate: + +```text +0 <= Report B <= Report A +derived_control = Report A - Report B +``` + +A negative derived control, Report B greater than Report A, mismatched report +definition, incompatible metric, missing saved definition, or non-zero excluded +traffic is a fail-closed reporting incident. Do not publish or interpret that +window; correct the inputs and rerun the complete pair after the same reporting +latency and invalid-traffic adjustment window. + +Use total metrics when the goal includes all GAM demand sources. GAM's +`Ad server impressions` and `Ad server clicks` metrics exclude Ad Exchange and +AdSense, so those narrower metrics should only be used when that exclusion is +intentional. GAM counts impressions and clicks according to its own tracking +rules; adding `ts=true` does not create new impression or click trackers. See +[Counting impressions and clicks](https://support.google.com/admanager/answer/2521337). + +Both reports must use the same metric names, and the target-network dry run must +prove that each metric is compatible with the chosen Enhanced or legacy +dimension and filters. Prefer non-targeted impression and click metrics because +`ts` is forbidden from line-item targeting; targeted metrics limited to keys +used for targeting do not represent this delivery-path cohort. If GAM cannot +produce an identical compatible metric in both reports, omit that metric rather +than substitute different definitions. Record the exact selected metric names +and successful dry-run exports before launch. + +### Descriptive rates for unequal cohort sizes + +The treatment cohort is intentionally small, so raw TS and production totals are +not directly comparable. GAM may show raw counts and descriptive normalized +rates where compatible metrics are available: + +- impressions per GAM ad request; +- fill rate; +- clicks per impression (CTR); and +- revenue per thousand impressions or requests. + +These rates describe GAM delivery; they do not estimate a causal treatment +effect. Impressions per routed pageview or per assigned visitor require a +denominator from the A/B router or site analytics because GAM cannot identify +unmarked production pageviews that made no ad request. Any causal analysis is +outside this implementation and requires a separately approved design defining +the eligible population, router/site denominators, analysis window, and stopping +rules. + +### Data-quality checks + +During the experiment, monitor: + +1. observed `ts=true` ad-request or impression share versus the router's + expected treatment allocation; +2. scheduled synthetic marker presence on initial, lazy, and refreshed treatment + requests; +3. scheduled synthetic marker absence on production requests; +4. non-experiment traffic served through Trusted Server; +5. unexpected `ts` values or line-item targeting; +6. report freshness and GAM invalid-traffic adjustments. + +A gap between expected and observed treatment share is a measurement incident, +not evidence of production performance, until missing-marker and request-volume +differences are ruled out. Because router assignment and GAM delivery normally +use page or visitor counts versus ad-request or impression counts, this share +comparison is a diagnostic rather than direct proof of marker coverage. A +request-correlated router or site denominator can strengthen the aggregate +coverage estimate, but without new request-correlated telemetry it still cannot +prove marker presence on every production request. + +Neither aggregate share nor a scheduled synthetic sample proves that every +production response or ad request carried the expected marker. Automated tests +establish code-path invariants; production checks provide sampled operational +evidence. The runbook must not describe either diagnostic as per-response +coverage telemetry. + +Checks 2–3 use a scheduled synthetic browser crawl of representative experiment +URLs. The crawler supplies known treatment and control cookies, captures GAM +network requests, and triggers initial, lazy, and refreshed slots. A failed +marker assertion is an operational measurement incident. This is external +validation rather than a site beacon or Trusted Server telemetry event; if the +experiment owner cannot operate the crawl, checks 2–3 become documented manual +samples and must not be represented as continuous production metrics. + +On treatment URLs with matched creative opportunities, the crawler must also +detect the fallback-only CSP state: capture CSP violations and verify that the +injected `adSlots`, `bids`, and initial `adInit` handoff executed. A page that +has `ts=true` only because the external bundle ran, while those inline scripts +were blocked, remains correctly marked as having a TS-emitted head but raises a +measurement incident. Since GAM cannot separate those requests afterward, the +incident owner must pause interpretation and exclude the affected time range +from both reports when clean boundaries can be established; otherwise the +experiment result is invalid. + +## Failure handling + +The marker is best-effort instrumentation and must never block ads or page +delivery. + +- If GPT never loads, there is no GAM request to classify. +- If attribution is disabled, the dormant gated bootstrap source may remain, but + no attribution callback is enqueued or executed and no activation flag or + attribute is emitted; GPT otherwise keeps its current behavior. +- If Trusted Server does not rewrite and emit a literal ``, or an in-scope + GPT request occurs before the injected head content runs, neither targeting + path can mark that request. Such traffic is ineligible for the experiment and + must be detected before launch or excluded from analysis. +- If an origin, decoder, or rewriter failure occurs after Fastly emitted the + rewritten head, the browser may already have queued `ts=true`. Any resulting + request remains treatment because the marker represents head delivery, not + full-response completion. The client may receive a truncated document; the + delivery failure is logged and investigated separately rather than being + reclassified as control. +- If CSP blocks Trusted Server's nonce-less inline scripts, the initial TS ad + stack is inert and the page is ineligible even when the first-party bundle + queues the attribution marker. The fallback prevents a page whose head was + emitted through Trusted Server from leaking into the inferred control cohort; + it does not make the deployment + healthy. If CSP blocks both inline scripts and the bundle, attribution also + fails. +- If `googletag.setConfig` is unavailable when a queued command runs, the + targeting step is a defensive no-op and must not throw. Supported treatment + deployments must use a GPT version with the configuration API; browser/GAM + validation detects an unsupported or missing API before experiment launch. +- If publisher code or a Trusted Server creative-opportunity targeting map + applies slot-level `ts`, GPT gives the slot-level value precedence. The + deployment audit prevents this collision; runtime filtering or interception is + out of scope because it could silently alter established targeting behavior. +- If Prebid `bidderSettings`, bidder or standard `adserverTargeting`, or + `setTargetingForGPTAsync` applies slot-level `ts`, the slot value can supersede + the page marker. The launch audit and characterization tests cover these paths; + Trusted Server does not filter or intercept them. +- If publisher code calls `setConfig({ targeting: null })`, sets `ts: null` or a + different value, calls legacy `pubads().clearTargeting()` for all keys or for + `ts`, or applies slot-level `ts`, the effective marker can be removed or + superseded. The publisher-code audit and refresh validation are required + because this design deliberately does not intercept those APIs. +- If the marker is absent on a treatment request, GAM classifies it with the + unmarked baseline. Coverage monitoring is the mitigation. +- GAM configuration or reporting failures do not affect ad serving. + +No retry, beacon, cookie read, backend request, or persistent client state is +added by this feature. + +## Privacy and consent + +`ts=true` contains no unique user identifier, cookie value, page URL, or auction +data. Its intended meaning is the head-delivery path of an eligible document; +outside that scope, copied activation is contamination rather than proof of +delivery. Because only the cookie-sticky treatment cohort is routed through +Trusted Server for this experiment, the value also reveals treatment-path +membership for that GAM request. It is therefore cohort information even though +it does not expose the assignment cookie or identify a person by itself. + +The implementation does not read the experiment cookie. Routing happens before +Trusted Server handles the request. Existing consent gates continue to decide +whether GAM requests or auctions occur. The marker does not create an ad request +that would otherwise be suppressed. Before enabling +`gam_attribution_enabled`, the experiment owner must complete the publisher's +privacy/data-governance review for sending this treatment-path attribute to GAM +and confirm that existing consent and data-use terms cover it. + +## Testing strategy + +### Automated tests + +1. Extend GPT configuration and head-insert tests for omitted, false, and true + `gam_attribution_enabled` values. Assert that the true case authorizes + page-level `ts=true` before the `ts.adInit` guard and any bootstrap `display()` + or `refresh()` call without changing the expected head-insert count; false + must preserve current output and behavior. +2. Extend the TSJS tag and HTML processor tests to prove the non-executable GPT + activation attribute appears only when GPT and attribution are both enabled + on the publisher-page bundle and adds no script tag. +3. Extend the existing Vitest/jsdom raw-bootstrap harness described above. + Exercise the + bootstrap with `googletag.setConfig` available, unavailable, and throwing, + and prove a later publisher callback still runs in every enabled case. Prove + the disabled case preserves existing behavior, then set `window.tsjs.adInit` + before evaluation and prove only the enabled marker still runs. +4. Extend the existing GPT bundle tests to prove module initialization queues + the fallback marker and remains non-blocking when `setConfig` is unavailable + or throws. +5. Retain assertions for `ts_initial=1` to prevent accidental replacement. +6. Retain refresh tests proving stale `ts_initial` and `hb_*` slot targeting is + cleared. Add an explicit assertion or source-level invariant that page-level + `ts` is not included in slot cleanup lists. +7. Extend creative-opportunity configuration tests to demonstrate that an + operator targeting map is forwarded verbatim, documenting why the deployment + audit must reject a configured `ts` key rather than assuming the client + overwrites or filters it. +8. Add Prebid characterization tests covering `bidderSettings`, custom + `adserverTargeting`, and `setTargetingForGPTAsync` so a generated slot-level + `ts` collision is visible and remains a launch-audit responsibility rather + than being silently filtered. +9. Extend no-post-processor streaming regression coverage to prove an + attribution-enabled Fastly response can emit the marked rewritten head before + origin EOF. Preserve the existing later-error/truncation behavior, + documenting that the marker does not certify complete response delivery and + adds no buffering. +10. Add focused Vitest/jsdom bundle-DOM coverage for the synchronous publisher + tag, proving `document.currentScript` is the attributed element and + false/non-publisher/duplicate-tag cases fail closed. Characterize a publisher + copying the attributed tag through `srcdoc` or `document.write`: the clone can + activate the fallback, so tests must not encode the false guarantee that only + independently rewritten nested documents can be marked. Manual browser/GAM + validation below covers the real deployed bundle without expanding the + repository's single-config Playwright harness. +11. Run the project-required target-matched Rust and JavaScript checks for the + touched files. + +### Browser/GAM validation + +Before experiment launch, with `gam_attribution_enabled = true` on the treatment +deployment: + +1. Load a treatment page using a known treatment cookie. +2. Confirm the initial in-scope GAM request contains `ts=true` using GPT + Publisher Console, Delivery Inspector, or the browser network panel. +3. Trigger a lazy slot and a refresh; confirm both requests still contain + `ts=true`. +4. Load the equivalent production page with a control cookie and confirm the key + is absent. +5. Disable `gam_attribution_enabled` on a Trusted Server validation deployment + and confirm GPT still functions while the inline flag, activation attribute, + and `ts` request key are absent. +6. Confirm `ts_initial=1` remains limited to its existing initial-slot + lifecycle. +7. Validate the deployed CSP by proving `adSlots`, the GPT bootstrap, `bids`, + and the initial `adInit` handoff execute on a representative page with + matched slots. A page that runs only the external bundle is ineligible even + if the fallback marker appears. +8. Set an unrelated page-level targeting key after `ts=true` and confirm both + keys remain on a later request. Treat an explicit page-level or per-key clear + as a failed publisher-code audit, not supported behavior. +9. Validate that IMA/video, direct-tag, server-side GAM, and nested GPT + inventory without an independently TS-rewritten document is absent from the + experiment and paired report scope. Directly validate any independently + rewritten nested documents that are intentionally included. +10. Run a short GAM report and verify treatment totals appear under `ts=true` + while overall totals remain unchanged apart from normal reporting latency. + +## Rollout + +1. Create the fixed reportable `ts=true` GAM key/value and record the intended + Enhanced or legacy reporting path, compatibility requirements, and billing + decision. +2. Freeze the eligible-scope manifest and draft the saved Report A/Report B + definitions and excluded-path queries from that manifest. +3. Audit response eligibility, including head rewrite/emission, ordering, CSP, + and publisher GPT calls that could precede or remove the marker. +4. Audit `ts` across publisher GPT code, Prebid-generated targeting, effective + `trusted-server.toml` creative-opportunity targeting maps, and every GAM + custom-targeting consumer. Retain the owner, evidence, timestamp, and re-audit + trigger. +5. Exclude IMA/video, direct-tag, server-side GAM, and nested GPT inventory not + independently routed, rewritten, and validated. Treat copied activation in an + excluded nested document as contamination. +6. Deploy `gam_attribution_enabled = true` while treatment routing remains + stopped. +7. Provision the scheduled synthetic crawl, assign an incident owner, and obtain + one successful treatment/control run covering initial, lazy, refreshed, CSP, + fallback, and disabled-setting checks. +8. On the validation deployment, validate treatment and control requests + manually, retain zero-count results for every excluded path, and save a short + paired-report dry run that proves the selected dimensions, filters, metrics, + and `0 <= Report B <= Report A` invariants. +9. Start the small cookie-sticky treatment cohort only after every configuration, + collision, privacy, CSP, synthetic, exclusion, and reporting gate passes. +10. Compare observed GAM treatment share with router allocation only as a + diagnostic before interpreting descriptive delivery results. + +Rollback is ordered so newly routed treatment traffic cannot become unmarked +control. First stop new treatment assignment/routing and verify through router +or access logs that routing stopped, then record the last clean reporting +boundary. Keep `gam_attribution_enabled = true` while already-open +documents—including long-lived SPA sessions and any marked document restored +from a cache—drain; they retain page-level targeting and may continue issuing +marked lazy or refreshed requests. Exclude the entire post-boundary drain +interval from both cohorts. The drain ends only after router/access logs and GAM +show no remaining `ts=true` traffic for one complete, runbook-defined reporting +interval. Then set `gam_attribution_enabled = false`, deploy the kill switch, +and use a fresh synthetic Trusted Server navigation to confirm normal GPT +behavior while the marker is absent. If marked traffic persists, keep +attribution enabled and the interval excluded rather than inferring it as +control. Historical GAM rows remain valid, and the GAM key may stay defined and +reportable for historical analysis. + +If an active privacy, targeting-collision, or ad-delivery incident requires an +immediate kill, deploy `gam_attribution_enabled = false` without waiting for +router verification. Treat the affected window and subsequent drain interval as +invalid for both cohorts, then stop and verify treatment routing and follow the +same drain-completion rule. The emergency path prioritizes serving safety over a +clean experiment boundary; it must never reinterpret newly unmarked treatment +traffic as control. + +## Alternatives considered + +### Reuse `ts_initial=1` + +Rejected because the key is slot-level, covers only TS-managed initial slots, +and is deliberately cleared on refresh. Changing its lifecycle would also break +its existing ownership semantics. + +### Add slot-level `ts=true` in `adInit` + +Rejected because it would miss publisher-owned or lazy slots that do not pass +through `adInit`, and existing refresh cleanup could remove it. It would measure +auction participation rather than page delivery. + +### Set `ts=true` only in the bootstrap + +Rejected as the sole path. Placing the enqueue before the existing `ts.adInit` +guard correctly handles a pre-installed ad-init implementation. The bundle is +not needed for that case. It remains useful when the inline script unexpectedly +stops executing but the synchronous first-party bundle still runs: without the +fallback, a treatment page with a TS-emitted head would be silently inferred as +control. +The fallback does not rescue the simultaneously blocked TS ad-stack scripts, so +that state is an incident rather than an eligible deployment mode. + +### Enable attribution whenever GPT is enabled + +Rejected because an upgrade would begin disclosing cohort/path information and +reserving the short `ts` key on every existing GPT deployment before its +collision, privacy, and reporting prerequisites were complete. The independent, +default-off setting permits deliberate rollout and rollback without removing the +GPT proxy, shim, or auction behavior. + +### Buffer the existing Fastly streaming HTML path until the complete rewrite succeeds + +Rejected because the marker must run from the rewritten head before publisher +GPT requests, while the no-post-processor Fastly path deliberately streams that +head before origin EOF. Adding full-response buffering to that path would change +latency, memory use, and first contentful paint. On that in-scope path, the +marker therefore certifies successful head rewrite and emission; later stream +completion is a separate delivery concern. Existing configurations that +register an HTML post-processor retain their current buffering behavior; this +feature adds none. + +### Use a longer descriptive key name such as `trusted_server` + +A descriptive name would lower the collision risk that the short `ts` key +carries. Rejected because the target-network provisioning preflight enforced a +10-character request-name limit and rejected `trusted_server` (14), even though +other GAM help surfaces describe a 20-character limit. The short `ts` name is +therefore mandatory for this deployment, and the cross-system collision audit +is the compensating control. Do not dual-write `ts` alongside any longer alias: +two names for one cohort would increase GAM setup and audit surface and permit +silent drift between reports. Only `ts=true` is valid. + +### Configure `ts=true` in creative-opportunity slot targeting + +Rejected because creative-opportunity targeting applies only to matched slots. +The experiment requirement covers every request after the enabled publisher +pipeline emits the document head for its local GPT PubAds service. + +### Rewrite `cust_params` on GAM network requests + +Rejected because it depends on GPT's internal request construction and encoding, +adds interception risk, and duplicates a supported GPT targeting API. + +### Mark production explicitly with `ts=false` + +Preferred in a fully controlled experiment, but unavailable because the +production path cannot be changed. The design documents the resulting unmarked +baseline limitation and requires coverage checks. + +## Acceptance criteria + +1. `gam_attribution_enabled` defaults to `false`. With omitted or false + configuration, dormant gated bootstrap source may remain present, but no + attribution callback is enqueued or executed and no inline attribution flag, + bundle activation attribute, or fallback is activated. Current GPT, shim, + `adInit`, and `ts_initial` behavior is preserved, and setting the field to + `false` independently disables attribution. +2. For an enabled deployment that satisfies the documented head-rewrite, + ordering, CSP, reserved-key, request-scope, and targeting-cleanup + prerequisites—and in which at least one callback successfully applies + page-level targeting before the first request, with no later clear or + override—every request from that document's local GPT PubAds service carries + `ts=true`, including initial, lazy, refreshed, publisher-owned, and SPA-route + requests. +3. For an in-scope document satisfying the non-cloned activation prerequisite, + the marker means the Trusted Server publisher pipeline rewrote and emitted + that document's head before the request. It does not certify complete streamed + response delivery. This feature adds no buffering; on the existing streaming + Fastly path, a later truncation does not reclassify an already marked request + as control. +4. Production/control pages remain unmodified and do not carry `ts` from this + feature. +5. `ts_initial=1` retains its current slot-level initial-request lifecycle, and + `ts` is not cleared by Prebid refresh or SPA slot cleanup. +6. No publisher code, Prebid-generated targeting, effective Trusted Server + creative-opportunity targeting map, or GAM custom-targeting consumer changes + ad eligibility, pricing, protection, routing, or the marker value because of + the measurement key. +7. The marker adds no unique identifier, cookie value, network request, + response buffering, or blocking work. Its disclosure of treatment-path + membership to GAM has passed the publisher's privacy/data-governance review. +8. The target-network preflight has fixed the only contract as `ts=true`; no + other value, alternative key name, alias, configurable marker, or dual-write + path exists. +9. The exact Enhanced or legacy reporting path, billing approval, dimensions, + filters, and compatible metrics pass a target-network dry run. Saved Report A + and Report B use the same frozen eligible-scope manifest and satisfy + `0 <= Report B <= Report A` for every interpreted metric; violations invalidate + the pair rather than being clamped or reported. +10. GAM output is described as delivery attribution, not a causal treatment + effect. Known treatment/control requests are sampled directly, while + aggregate marker share is compared diagnostically with router allocation; + neither check is represented as proof of per-response production coverage. +11. Every excluded traffic path has a retained zero-count preflight or an + independent filter applied identically to both saved reports. +12. Nested inventory is included only when independently routed, rewritten, and + validated. Because the activation tag is clonable, marker presence alone is + not proof of eligibility; copied activation in excluded inventory is a + contamination incident. +13. CSP-compatible inline execution is proven before launch. A fallback-only + page remains attributed to treatment but raises an incident and cannot be + treated as a healthy experiment page. +14. Normal rollback stops and verifies new treatment routing, records the clean + boundary, and keeps attribution enabled until already-open or cached marked + documents drain according to the documented rule. Only then is the kill + switch deployed. An emergency kill invalidates the affected and drain + windows instead of treating newly unmarked traffic as control. diff --git a/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md b/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md new file mode 100644 index 000000000..1c73fd668 --- /dev/null +++ b/docs/superpowers/specs/2026-07-20-ssat-debug-comment-config-design.md @@ -0,0 +1,488 @@ +# SSAT Debug Comment Configuration Design + +**Date:** 2026-07-20 + +**Status:** Proposed security revision; awaiting approval + +**Issue:** [IABTechLab/trusted-server#935](https://github.com/IABTechLab/trusted-server/issues/935) — "For SSAT, make debug comment configurable" + +## Summary + +The server-side auction template (SSAT) can inject a `` +HTML comment before the bids `