#345: narrow HEAD+Range shim so DuckDB-WASM uses range requests (74 MB -> 3 MB cold load)

.github/workflows/data-origin-contract.yml

@@ -0,0 +1,32 @@
+# Cheap HTTP contract check for data.isamples.org — the gate that would have
+# caught #345 (DuckDB-WASM silently downloading whole files). No browser, no
+# DuckDB, a few KB on the wire. Runs on every push to main, on PRs that touch the
+# Worker or the test, and daily — the origin can regress without a commit.
+name: data-origin-contract
+on:
+  push:
+    branches: [main]
+  pull_request:
+    paths:
+      - 'workers/data-isamples-org/**'
+      - 'tests/test_data_origin_contract.py'
+      - 'isamples_202608_release_manifest.json'
+      - '.github/workflows/data-origin-contract.yml'
+  schedule:
+    - cron: '17 6 * * *'   # daily 06:17 UTC
+  workflow_dispatch:
+jobs:
+  contract:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: pip install --quiet pytest requests
+      - name: HEAD/Range/CORS/manifest contract against the live origin
+        env:
+          ISAMPLES_DATA_ORIGIN: https://data.isamples.org
+        # --runxfail kept deliberately: any future xfail on a contract test must still fail CI.
+        run: pytest -q --runxfail tests/test_data_origin_contract.py

tests/test_data_origin_contract.py

@@ -0,0 +1,170 @@
+"""
+HTTP contract tests for the data origin (`data.isamples.org`).
+
+WHY THIS FILE EXISTS
+--------------------
+The Explorer's whole architecture rests on one assumption: DuckDB-WASM fetches
+only the byte ranges a query touches. In August 2026 we discovered that had been
+silently false — a cold load transferred **~74 MB before anything was usable, instead
+of ~3.5 MB** because
+DuckDB's range-support probe was answered in a way it did not accept, so it fell
+back to downloading whole files (#345).
+
+That regression was invisible to every existing test. It was also *mis-cleared*
+in June (`ISSUE_313_FINDINGS_2026-06-26.md`) by a `curl` probe that used **GET**
+where DuckDB uses **HEAD** — the check looked right and proved nothing.
+
+These tests are deliberately cheap: no browser, no DuckDB, a handful of bytes on
+the wire. They run before the Playwright smoke gate so this class of failure is
+caught before anything downloads 74 MB to discover it.
+
+MAINTENANCE NOTE
+----------------
+`tools/build_release_manifest.py` performs the ranged-GET probe but NOT the HEAD
+probe — which is exactly why it could report a healthy origin while DuckDB was
+broken. If you touch either, keep them in sync.
+"""
+import json
+import os
+import pathlib
+
+import pytest
+import requests
+
+ORIGIN = os.environ.get("ISAMPLES_DATA_ORIGIN", "https://data.isamples.org")
+PAGE_ORIGIN = "https://isamples.org"
+MANIFEST = pathlib.Path(__file__).resolve().parent.parent / "isamples_202608_release_manifest.json"
+
+# The Worker 403s some default user agents; identify honestly.
+UA = {"User-Agent": "isamples-ci-contract/1.0 (+https://isamples.org)"}
+TIMEOUT = 30
+
+
+def _manifest_files():
+    if not MANIFEST.exists():
+        pytest.skip(f"release manifest not found at {MANIFEST}")
+    return json.loads(MANIFEST.read_text())["files"]
+
+
+def _boot_critical_large_file():
+    """The biggest boot-critical parquet — the one a full read actually hurts.
+
+    Skips (rather than fails) if the origin does not serve it. Without this a
+    404 from, say, a partially-seeded local test origin gets reported as
+    "the shim has widened", which is a confidently wrong diagnosis — the exact
+    failure mode this whole file exists to prevent.
+    """
+    files = _manifest_files()
+    name = None
+    for candidate in ("isamples_202608_samples_map_lite_v3.parquet",
+                      "isamples_202608_sample_facet_masks.parquet"):
+        if candidate in files:
+            name = candidate
+            break
+    if name is None:
+        name = max(files.items(), key=lambda kv: kv[1].get("size_bytes", 0))[0]
+
+    probe = requests.head(f"{ORIGIN}/{name}", headers=UA, timeout=TIMEOUT)
+    if probe.status_code == 404:
+        pytest.skip(f"{ORIGIN} does not serve {name} (404) — not a contract failure")
+    return name, files[name]["size_bytes"]
+
+
+def test_ranged_get_returns_206_with_exact_content_range():
+    """The load-bearing property: partial GETs work and report the full size.
+
+    This one passes today and has always passed — which is precisely why it was
+    not enough on its own. Keep it: if it ever breaks, range reads are dead.
+    """
+    name, size = _boot_critical_large_file()
+    r = requests.get(f"{ORIGIN}/{name}",
+                     headers={**UA, "Range": "bytes=0-0", "Origin": PAGE_ORIGIN},
+                     timeout=TIMEOUT)
+    assert r.status_code == 206, f"ranged GET returned {r.status_code}, not 206"
+    assert r.headers.get("Content-Range") == f"bytes 0-0/{size}", (
+        f"Content-Range {r.headers.get('Content-Range')!r} disagrees with the "
+        f"manifest size {size}")
+    assert len(r.content) == 1, f"ranged GET returned {len(r.content)} bytes, expected 1"
+
+
+def test_cors_exposes_headers_the_explorer_must_read():
+    """Cross-origin JS cannot see Content-Range/Accept-Ranges unless exposed."""
+    name, _ = _boot_critical_large_file()
+    r = requests.get(f"{ORIGIN}/{name}",
+                     headers={**UA, "Range": "bytes=0-0", "Origin": PAGE_ORIGIN},
+                     timeout=TIMEOUT)
+    exposed = (r.headers.get("Access-Control-Expose-Headers") or "").lower()
+    for h in ("content-range", "accept-ranges", "content-length"):
+        assert h in exposed, f"{h} not in Access-Control-Expose-Headers ({exposed!r})"
+    assert r.headers.get("Access-Control-Allow-Origin") in ("*", PAGE_ORIGIN)
+
+
+def test_duckdb_124_head_range_compatibility():
+    # The #345 shim went live on data.isamples.org on 2026-08-27 (Worker version
+    # 6f8170a7…); this test is now a hard gate. If it fails, DuckDB-WASM is back to
+    # downloading whole files (~74 MB before the facets are usable) — see #345/#351.
+    """DuckDB-WASM 1.24.0's range-support probe must be answered with 206.
+
+    *** THIS ASSERTS A DELIBERATE STANDARDS DIVERGENCE. ***
+
+    RFC 9110 section 14.2: Range is defined only for GET, and a server MUST
+    IGNORE it on other methods including HEAD. A plain 200 here is the CORRECT
+    HTTP answer. But DuckDB-WASM 1.24.0 — the version Quarto's OJS runtime pins
+    — probes with exactly `HEAD` + `Range: bytes=0-` and treats anything other
+    than 206 as "this server cannot do partial reads", then downloads whole
+    files.
+
+    So this test encodes a compatibility shim, not correct HTTP. It should be
+    DELETED, along with the Worker shim, once the Explorer no longer depends on
+    that probe (i.e. when it stops using the pinned duckdb-wasm and does its own
+    init on a conformant version). See #345.
+    """
+    name, size = _boot_critical_large_file()
+    r = requests.head(f"{ORIGIN}/{name}",
+                      headers={**UA, "Range": "bytes=0-", "Origin": PAGE_ORIGIN},
+                      timeout=TIMEOUT)
+    assert r.status_code == 206, (
+        f"HEAD+Range returned {r.status_code}. DuckDB-WASM will fall back to full "
+        f"HTTP reads and the Explorer will transfer tens of MB on cold load.")
+    assert r.headers.get("Content-Range") == f"bytes 0-{size - 1}/{size}"
+    assert r.headers.get("Content-Length") == str(size)
+    assert not r.content, "HEAD must not return a body"
+
+
+def test_shim_does_not_widen_to_other_ranged_heads():
+    """The #345 shim must stay scoped to the exact probe shape.
+
+    Any OTHER ranged HEAD must remain standards-correct (200, no Content-Range),
+    so the divergence cannot leak to other clients or harden into a contract we
+    did not intend to offer.
+    """
+    name, _ = _boot_critical_large_file()
+    for rng in ("bytes=0-99", "bytes=100-199", "bytes=-100"):
+        r = requests.head(f"{ORIGIN}/{name}",
+                          headers={**UA, "Range": rng, "Origin": PAGE_ORIGIN},
+                          timeout=TIMEOUT)
+        assert r.status_code == 200, (
+            f"HEAD with {rng!r} returned {r.status_code}; the #345 shim has widened "
+            f"beyond the single DuckDB probe shape and is now diverging from RFC 9110 "
+            f"more than intended")
+        assert "Content-Range" not in r.headers, (
+            f"HEAD with {rng!r} carried a Content-Range; see above")
+
+
+def test_manifest_sizes_match_the_origin():
+    """Catches the data/doc drift class: manifest says one size, origin serves another."""
+    files = _manifest_files()
+    checked = 0
+    for name, meta in files.items():
+        if not name.endswith(".parquet") or "/" in name:
+            continue
+        size = meta.get("size_bytes")
+        if not size:
+            continue
+        r = requests.head(f"{ORIGIN}/{name}", headers=UA, timeout=TIMEOUT)
+        assert r.status_code == 200, f"{name}: HEAD returned {r.status_code}"
+        assert r.headers.get("Content-Length") == str(size), (
+            f"{name}: manifest says {size} bytes, origin serves "
+            f"{r.headers.get('Content-Length')}")
+        checked += 1
+    assert checked > 0, "no parquet entries checked — manifest shape may have changed"

workers/data-isamples-org/deploy-canary.sh

@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+# Deploy the #345 HEAD+Range shim to a NON-PRODUCTION canary and verify it.
+#
+# Safe by construction: uses wrangler.canary.toml, which has a different Worker
+# name and NO routes, so it cannot land in front of data.isamples.org. The
+# production wrangler.toml is never read or modified.
+#
+#   ./deploy-canary.sh            # deploy + verify + print the test URL
+#   ./deploy-canary.sh --verify   # verify an already-deployed canary
+#   ./deploy-canary.sh --teardown # delete the canary
+set -uo pipefail
+cd "$(dirname "$0")"
+
+NAME="isamples-data-345-canary"
+PROBE_FILE="isamples_202608_h3_summary_res4.parquet"
+PROBE_SIZE=505651
+STAGING="https://rdhyee.github.io/isamplesorg.github.io"
+
+if [ "${1:-}" = "--teardown" ]; then
+  echo "Deleting canary Worker '$NAME'..."
+  npx wrangler delete --name "$NAME"
+  exit $?
+fi
+
+# --- auth -------------------------------------------------------------------
+if ! npx wrangler whoami >/dev/null 2>&1; then
+  echo "Not logged in to Cloudflare."
+  echo "Run this in an interactive terminal first:"
+  echo
+  echo "    npx wrangler login"
+  echo
+  echo "(or export CLOUDFLARE_API_TOKEN=... with Workers Scripts:Edit + R2 read)"
+  exit 1
+fi
+
+# --- deploy -----------------------------------------------------------------
+if [ "${1:-}" != "--verify" ]; then
+  echo "==> Deploying canary (NO routes, workers.dev only)"
+  # Abort explicitly on a failed deploy. This script deliberately does not use
+  # `set -e` (several grep pipelines below are allowed to fail), so without this
+  # check a failed upload would fall through and verify an OLDER canary, then
+  # report success — the wrong answer with a confident face.
+  if ! npx wrangler deploy -c wrangler.canary.toml | tee /tmp/canary_deploy.log; then
+    echo "!! wrangler deploy failed — nothing verified, nothing promoted."
+    exit 1
+  fi
+  echo
+fi
+
+# The deploy output contains the workers.dev URL; recover it, else construct it.
+URL=$(grep -oE 'https://[a-z0-9._-]*\.workers\.dev' /tmp/canary_deploy.log 2>/dev/null | head -1)
+if [ -z "$URL" ]; then
+  SUB=$(npx wrangler whoami 2>/dev/null | grep -oE '[a-z0-9-]+\.workers\.dev' | head -1)
+  [ -n "$SUB" ] && URL="https://${NAME}.${SUB}"
+fi
+if [ -z "$URL" ]; then
+  echo "!! Could not determine the canary URL. Check /tmp/canary_deploy.log and pass it manually:"
+  echo "   CANARY_URL=https://... $0 --verify"
+  URL="${CANARY_URL:-}"
+  [ -z "$URL" ] && exit 1
+fi
+
+echo "==> Canary URL: $URL"
+echo
+
+# --- wait for the workers.dev route to go live ------------------------------
+# A freshly deployed workers.dev hostname is NOT immediately routable: for the
+# first few seconds Cloudflare's edge answers 404 before the route propagates.
+# The first version of this script verified instantly and reported seven
+# confident failures against a Worker that was in fact perfectly healthy —
+# including "the shim has widened", which was simply untrue. Poll for readiness
+# before asserting anything.
+echo "==> Waiting for the workers.dev route to propagate"
+ready=0
+for i in $(seq 1 30); do
+  code=$(curl -s --max-time 10 -o /dev/null -w '%{http_code}' "$URL/")
+  if [ "$code" = "200" ]; then
+    echo "  route live after ~$((i*2))s"
+    ready=1
+    break
+  fi
+  sleep 2
+done
+if [ "$ready" -ne 1 ]; then
+  echo "  !! route still not answering 200 at $URL/ after ~60s."
+  echo "     Not asserting anything — a 404 here means 'not deployed yet',"
+  echo "     not 'the change is broken'. Re-run with --verify shortly."
+  exit 1
+fi
+echo
+
+# --- verify the handshake actually changed ---------------------------------
+fail=0
+ok() { printf "  ok   %s\n" "$1"; }
+no() { printf "  FAIL %s (%s)\n" "$1" "$2"; fail=1; }
+
+echo "==> Verifying the DuckDB probe now gets 206"
+S=$(curl -s --max-time 20 -o /dev/null -w '%{http_code}' -I -H 'Range: bytes=0-' "$URL/$PROBE_FILE")
+CR=$(curl -s --max-time 20 -I -H 'Range: bytes=0-' "$URL/$PROBE_FILE" | grep -i '^content-range:' | tr -d '\r' | cut -d' ' -f2-)
+[ "$S" = "206" ] && ok "HEAD+Range 'bytes=0-' -> 206" || no "HEAD+Range -> 206" "got $S"
+[ "$CR" = "bytes 0-$((PROBE_SIZE-1))/$PROBE_SIZE" ] && ok "Content-Range correct" \
+  || no "Content-Range" "got '$CR'"
+# DuckDB-WASM accepts the probe on 206 + a usable Content-Length, so assert the
+# length too (a 206 with a wrong/missing length still means whole-file reads).
+CL=$(curl -s --max-time 20 -I -H 'Range: bytes=0-' "$URL/$PROBE_FILE" | grep -i '^content-length:' | tr -d '\r' | awk '{print $2}')
+[ "$CL" = "$PROBE_SIZE" ] && ok "Content-Length == $PROBE_SIZE" || no "Content-Length" "got '$CL'"
+
+echo
+echo "==> Verifying the shim did NOT widen (these must stay standards-correct 200)"
+for R in 'bytes=0-99' 'bytes=100-199' 'bytes=-100'; do
+  S=$(curl -s --max-time 20 -o /dev/null -w '%{http_code}' -I -H "Range: $R" "$URL/$PROBE_FILE")
+  [ "$S" = "200" ] && ok "HEAD '$R' -> 200" || no "HEAD '$R' -> 200" "got $S"
+done
+
+echo
+echo "==> Verifying GET paths unchanged"
+S=$(curl -s --max-time 20 -o /dev/null -w '%{http_code}' -H 'Range: bytes=0-99' "$URL/$PROBE_FILE")
+[ "$S" = "206" ] && ok "ranged GET -> 206" || no "ranged GET -> 206" "got $S"
+S=$(curl -s --max-time 20 -o /dev/null -w '%{http_code}' "$URL/$PROBE_FILE")
+[ "$S" = "200" ] && ok "plain GET -> 200" || no "plain GET -> 200" "got $S"
+
+echo
+if [ "$fail" -ne 0 ]; then
+  echo "VERIFICATION FAILED — do not promote to the data.isamples.org route."
+  exit 1
+fi
+echo "All canary checks passed."
+echo
+echo "───────────────────────────────────────────────────────────────"
+echo "Open the staging Explorer against the canary:"
+echo
+echo "  $STAGING/explorer.html?data_base=$URL"
+echo
+echo "Measure it end-to-end (from the repo root):"
+echo
+echo "  python tests/playwright/bandwidth_matrix.py $STAGING \\"
+echo "      --profiles unthrottled,3g-fast --budget 600 \\"
+echo "      --query \"?data_base=$URL\""
+echo
+echo "Expect: ~3.5 MB until the facets are usable (instead of ~74 MB) and 0 'full HTTP read'"
+echo "fallbacks. NOTE the harness stops at the milestones: the full boot still streams the 63 MB"
+echo "map file afterwards — twice, via range reads — so the 60 s total is ~130 MB (#351)."
+echo
+echo "Tear down when done:   $0 --teardown"
+echo "───────────────────────────────────────────────────────────────"