[docs-agent] Add Try It method audit (DOCS-101)

audits/2026-05-29-try-it-audit/README.md

@@ -0,0 +1,32 @@
+# Try It audit — May 2026
+
+Full automated audit of every method's canonical example payload across the OpenRPC specs in `src/openrpc/`. Probes each example live against the documented mainnet endpoint with the `docs-demo` API key (using the same `Origin` / `Referer` headers Fern's Try It widget sends) and catalogs every failure.
+
+* [report.md](./report.md) — full categorized report with per-category breakdown, per-spec failure rates, and top quick-win recommendations.
+* [failures.csv](./failures.csv) — every failed probe as a row (`spec`, `method`, `category`, `http_status`, `rpc_code`, `rpc_message`) for spreadsheet pivot analysis.
+* [scripts/](./scripts) — the Python scripts that produced the report. Reproducible from a fresh `pnpm install && pnpm run generate:rpc` checkout.
+
+## Headline numbers
+
+* Probed: **3,749** methods (chain specs + Alchemy product specs)
+* OK: **2,604**
+* Failed: **842** (24.4%)
+* Skipped (websocket-only or no example in the spec): **303**
+
+## Reproducing
+
+```
+pnpm install
+pnpm run generate:rpc                  # produces content/api-specs/
+python3 scripts/run_audit.py            # initial pass; writes results.json
+python3 scripts/retry_429.py "HTTP 429" # retry rate-limited probes
+python3 scripts/build_report.py         # rebuilds report.md and failures.csv
+```
+
+`docs-demo` is aggressively rate-limited; the retry script paces requests serially per host.
+
+## Out of scope
+
+* OpenAPI REST specs (`src/openapi/`). Each REST spec needs a per-spec adapter (path params, query params, auth, body shapes vary). Worth a follow-up audit.
+* Production logs of customer-hit errors. Needs data-team observability access.
+* Fern Try It auto-fill on optional fields not in the example. The audit only sends the literal example payload, so the failure count here is a lower bound on what users actually see in the widget.

audits/2026-05-29-try-it-audit/scripts/reprobe_overrides.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Re-probe specific specs with a corrected server override and merge back into results.json."""
+import asyncio
+import json
+import sys
+import time
+from pathlib import Path
+from urllib.parse import urlparse
+import httpx
+
+REPO = Path("/root/docs-repo")
+OUT = Path("/root/audit")
+SPECS_DIR = REPO / "content/api-specs"
+
+OVERRIDES = {
+    "wallet-api": "https://api.g.alchemy.com/v2",
+}
+
+
+def build_params(method):
+    examples = method.get("examples") or []
+    if not examples:
+        return None, "no example"
+    ex = examples[0]
+    if not isinstance(ex, dict):
+        return None, "example not object"
+    ex_params = ex.get("params")
+    if ex_params is None:
+        return None, "example missing params"
+    ps = method.get("paramStructure", "by-position")
+    if ps == "by-name":
+        out = {}
+        for p in ex_params:
+            if isinstance(p, dict) and "value" in p and p.get("name") is not None:
+                out[p["name"]] = p["value"]
+        return out, None
+    else:
+        out = [p["value"] if (isinstance(p, dict) and "value" in p) else p for p in ex_params]
+        return out, None
+
+
+async def main():
+    with (OUT / "results.json").open() as f:
+        R = json.load(f)
+
+    # Build re-probe list from spec
+    new_probes = []
+    for spec_name, server_url in OVERRIDES.items():
+        # Try both alchemy/json-rpc and chains paths
+        candidates = [
+            SPECS_DIR / f"alchemy/json-rpc/{spec_name}.json",
+            SPECS_DIR / f"chains/{spec_name}.json",
+        ]
+        spec_path = next((p for p in candidates if p.exists()), None)
+        if not spec_path:
+            print(f"spec not found: {spec_name}", file=sys.stderr)
+            continue
+        with spec_path.open() as f:
+            d = json.load(f)
+        url = server_url.rstrip("/") + "/docs-demo"
+        for method in d.get("methods", []):
+            name = method.get("name")
+            if not name:
+                continue
+            params, err = build_params(method)
+            new_probes.append({
+                "spec": spec_name,
+                "method": name,
+                "params": params,
+                "paramStructure": method.get("paramStructure", "by-position"),
+                "url": url,
+                "skipped_reason": err,
+            })
+
+    print(f"Re-probing {len(new_probes)} methods on overridden endpoints", file=sys.stderr)
+
+    async def one(client, p):
+        if p["skipped_reason"]:
+            return None  # leave existing entry alone
+        body = {"jsonrpc": "2.0", "id": 1, "method": p["method"], "params": p["params"]}
+        headers = {
+            "Content-Type": "application/json",
+            "Origin": "https://www.alchemy.com",
+            "Referer": "https://www.alchemy.com/",
+        }
+        backoff = 3.0
+        for attempt in range(4):
+            try:
+                r = await client.post(p["url"], json=body, headers=headers, timeout=30.0)
+            except Exception as e:
+                return {"status": "fail", "reason": f"transport {type(e).__name__}", "rpc_error": str(e)[:200], "http_status": None}
+            if r.status_code == 429 and attempt < 3:
+                await asyncio.sleep(backoff)
+                backoff *= 1.8
+                continue
+            if r.status_code != 200:
+                return {"status": "fail", "reason": f"HTTP {r.status_code}", "http_status": r.status_code, "rpc_error": (r.text or "")[:300]}
+            try:
+                j = r.json()
+            except Exception:
+                return {"status": "fail", "reason": "non-JSON response", "http_status": 200, "rpc_error": (r.text or "")[:300]}
+            if "error" in j and j["error"] is not None:
+                e = j["error"]
+                return {"status": "fail", "reason": "jsonrpc error", "http_status": 200, "rpc_error": {
+                    "code": e.get("code") if isinstance(e, dict) else None,
+                    "message": (e.get("message") if isinstance(e, dict) else str(e))[:300],
+                    "data": (str(e.get("data"))[:300] if isinstance(e, dict) and e.get("data") is not None else None),
+                }}
+            if "result" in j:
+                return {"status": "ok", "http_status": 200}
+            return {"status": "fail", "reason": "no result, no error", "http_status": 200, "rpc_error": (r.text or "")[:300]}
+        return {"status": "fail", "reason": "HTTP 429 after retry", "http_status": 429, "rpc_error": ""}
+
+    async with httpx.AsyncClient() as client:
+        sem = asyncio.Semaphore(2)
+        async def worker(p):
+            async with sem:
+                await asyncio.sleep(0.6)  # gentle pacing
+                return await one(client, p)
+        outcomes = await asyncio.gather(*(worker(p) for p in new_probes))
+
+    # Merge: update entries in R matching (spec, method)
+    by_key = {(r["spec"], r["method"]): i for i, r in enumerate(R)}
+    updates = 0
+    for p, outcome in zip(new_probes, outcomes):
+        if outcome is None:
+            continue
+        key = (p["spec"], p["method"])
+        idx = by_key.get(key)
+        if idx is None:
+            continue
+        existing = R[idx]
+        # Update URL too, since we changed the target
+        existing.update({k: outcome[k] for k in outcome})
+        existing["url"] = p["url"]
+        existing["params"] = p["params"]
+        existing["paramStructure"] = p["paramStructure"]
+        # Clear conflicting old fields
+        if outcome.get("status") == "ok":
+            existing.pop("reason", None)
+            existing.pop("rpc_error", None)
+        updates += 1
+
+    with (OUT / "results.json").open("w") as f:
+        json.dump(R, f, indent=2)
+    print(f"Updated {updates} entries", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

audits/2026-05-29-try-it-audit/scripts/retry_429.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Retry pass for probes that returned HTTP 429 (rate-limited).
+
+Slower per-host rate (1 RPS each) with exponential backoff on repeated 429s.
+Replaces matching entries in results.json with the retry outcome.
+"""
+
+import asyncio
+import json
+import sys
+import time
+from pathlib import Path
+from urllib.parse import urlparse
+
+import httpx
+
+OUT = Path("/root/audit")
+
+
+class HostLimiter:
+    def __init__(self, min_interval=1.0):
+        self._sems = {}
+        self._last = {}
+        self._lock = asyncio.Lock()
+        self.min_interval = min_interval
+
+    def sem(self, host):
+        if host not in self._sems:
+            self._sems[host] = asyncio.Semaphore(1)  # serial per host
+        return self._sems[host]
+
+    async def wait(self, host):
+        async with self._lock:
+            last = self._last.get(host, 0)
+            now = time.monotonic()
+            delta = now - last
+            if delta < self.min_interval:
+                await asyncio.sleep(self.min_interval - delta)
+            self._last[host] = time.monotonic()
+
+
+async def one(client, limiter, probe, max_retries=4):
+    if "skipped" in probe:
+        return {**probe, "status": "skipped"}
+    host = urlparse(probe["url"]).hostname
+    sem = limiter.sem(host)
+    body = {
+        "jsonrpc": "2.0", "id": 1,
+        "method": probe["method"], "params": probe["params"],
+    }
+    headers = {
+        "Content-Type": "application/json",
+        "Origin": "https://www.alchemy.com",
+        "Referer": "https://www.alchemy.com/",
+    }
+    backoff = 4.0
+    for attempt in range(max_retries):
+        async with sem:
+            await limiter.wait(host)
+            try:
+                r = await client.post(probe["url"], json=body, headers=headers, timeout=30.0)
+            except httpx.TimeoutException:
+                return {**probe, "status": "fail", "reason": "timeout", "http_status": None, "rpc_error": None}
+            except httpx.ConnectError as e:
+                return {**probe, "status": "fail", "reason": "dns/connect", "http_status": None, "rpc_error": str(e)[:200]}
+            except Exception as e:
+                return {**probe, "status": "fail", "reason": f"transport {type(e).__name__}", "http_status": None, "rpc_error": str(e)[:200]}
+        if r.status_code == 429:
+            if attempt < max_retries - 1:
+                await asyncio.sleep(backoff)
+                backoff *= 1.8
+                continue
+            return {**probe, "status": "fail", "reason": "HTTP 429 after retry", "http_status": 429, "rpc_error": (r.text or "")[:200]}
+        if r.status_code != 200:
+            return {**probe, "status": "fail", "reason": f"HTTP {r.status_code}", "http_status": r.status_code, "rpc_error": (r.text or "")[:300]}
+        try:
+            j = r.json()
+        except Exception:
+            return {**probe, "status": "fail", "reason": "non-JSON response", "http_status": 200, "rpc_error": (r.text or "")[:300]}
+        if "error" in j and j["error"] is not None:
+            err = j["error"]
+            return {**probe, "status": "fail", "reason": "jsonrpc error", "http_status": 200, "rpc_error": {
+                "code": err.get("code") if isinstance(err, dict) else None,
+                "message": (err.get("message") if isinstance(err, dict) else str(err))[:300],
+                "data": (str(err.get("data"))[:300] if isinstance(err, dict) and err.get("data") is not None else None),
+            }}
+        if "result" in j:
+            return {**probe, "status": "ok", "http_status": 200}
+        return {**probe, "status": "fail", "reason": "no result, no error", "http_status": 200, "rpc_error": (r.text or "")[:300]}
+
+
+def select_targets(results, modes):
+    """Filter the indices of the results entries that match modes (list of reasons)."""
+    idxs = []
+    for i, r in enumerate(results):
+        if r.get("status") == "fail" and r.get("reason") in modes:
+            idxs.append(i)
+    return idxs
+
+
+async def main(modes):
+    with (OUT / "results.json").open() as f:
+        results = json.load(f)
+    target_idxs = select_targets(results, modes)
+    print(f"Retry targets: {len(target_idxs)} of {len(results)}", file=sys.stderr)
+
+    limiter = HostLimiter(min_interval=2.0)
+    limits = httpx.Limits(max_connections=120, max_keepalive_connections=60)
+
+    async with httpx.AsyncClient(limits=limits) as client:
+        # Build fresh probes for these (need to reconstruct the original dict
+        # since the run_audit results carry the relevant fields already).
+        sem_global = asyncio.Semaphore(60)
+        async def worker(idx):
+            async with sem_global:
+                probe = {
+                    "spec": results[idx]["spec"],
+                    "category": results[idx]["category"],
+                    "method": results[idx]["method"],
+                    "params": results[idx]["params"],
+                    "paramStructure": results[idx].get("paramStructure"),
+                    "url": results[idx]["url"],
+                }
+                outcome = await one(client, limiter, probe)
+                results[idx] = outcome
+                return idx
+
+        tasks = [asyncio.create_task(worker(i)) for i in target_idxs]
+        done = 0
+        total = len(tasks)
+        for fut in asyncio.as_completed(tasks):
+            await fut
+            done += 1
+            if done % 200 == 0 or done == total:
+                print(f"  {done}/{total}", file=sys.stderr)
+
+    with (OUT / "results.json").open("w") as f:
+        json.dump(results, f, indent=2)
+    print(f"Updated {OUT/'results.json'}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    modes = sys.argv[1:] or ["HTTP 429"]
+    asyncio.run(main(modes))