From 79447e311807a03639eded8fadf8d7467eb9035e Mon Sep 17 00:00:00 2001 From: Dan Baciu Date: Fri, 21 Aug 2026 18:49:37 +0200 Subject: [PATCH] feat(sleep): paired A/B evalkit with McNemar and bootstrap CIs Add a stdlib evalkit so Sleep comparisons share one instrument: one fixed task manifest, McNemar on paired binary outcomes, percentile bootstrap CIs on the success-rate delta, and multi-seed variance bands. Cross-manifest id mismatches are refused. The nightly gate is unchanged. Related: #108 --- CHANGELOG.md | 4 + docs/reference/cli.md | 7 +- docs/sleep/README.md | 14 + docs/sleep/evalkit.md | 53 ++ skillopt_sleep/__main__.py | 26 + skillopt_sleep/evalkit.py | 479 ++++++++++++++++++ tests/fixtures/evalkit/aa_manifest.json | 8 + tests/fixtures/evalkit/aa_outcomes.json | 9 + tests/fixtures/evalkit/mcnemar_textbook.json | 10 + .../evalkit/results_searchqa_nano_gated.json | 9 + tests/test_evalkit.py | 214 ++++++++ 11 files changed, 831 insertions(+), 2 deletions(-) create mode 100644 docs/sleep/evalkit.md create mode 100644 skillopt_sleep/evalkit.py create mode 100644 tests/fixtures/evalkit/aa_manifest.json create mode 100644 tests/fixtures/evalkit/aa_outcomes.json create mode 100644 tests/fixtures/evalkit/mcnemar_textbook.json create mode 100644 tests/fixtures/evalkit/results_searchqa_nano_gated.json create mode 100644 tests/test_evalkit.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 513840a9..3da89ab6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ All notable changes to SkillOpt are documented here. This project adheres to ## [Unreleased] ### Added +- **SkillOpt-Sleep paired A/B evalkit** (`python -m skillopt_sleep.evalkit`): + McNemar plus percentile-bootstrap CIs on a fixed task manifest, with + multi-seed variance bands and an A/A calibration. The nightly gate is + unchanged (thanks @bogdanbaciu21). - **SkillOpt-Sleep multi-skill fan-out and reviewed subset adoption**: each hinted skill is consolidated from its own pinned live baseline, staged as an independent proposal with per-skill gate evidence, and promoted only through diff --git a/docs/reference/cli.md b/docs/reference/cli.md index f0ea40a5..6ed591c2 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -123,8 +123,11 @@ skillopt-sleep [options] python -m skillopt_sleep [options] ``` -Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, and -`unschedule`. Common options include: +Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, +`unschedule`, and `evalkit`. `evalkit` is also available as +`python -m skillopt_sleep.evalkit` and compares two conditions on one fixed +task manifest (McNemar + bootstrap CI). See `docs/sleep/evalkit.md`. Common +options for the nightly actions include: | Argument | Description | |---|---| diff --git a/docs/sleep/README.md b/docs/sleep/README.md index 2576c127..c3c00db2 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -339,6 +339,20 @@ correctness signal; the validation gate still governs what ships. | `recall_k` | `0` | Associative recall — pull the K most-similar past tasks (from a persisted archive) into tonight's dream. | | `dream_factor` | `0` | Add N lightweight synthetic variants of each task. | +### Paired A/B evalkit + +Reports and PRs that claim "B beats A" should go through the shared evalkit +rather than quoting a single-run cell. One command pairs two conditions on one +fixed task manifest, runs McNemar's test, and reports a bootstrap CI on the +success-rate delta. The nightly gate is unchanged. + +```text +python -m skillopt_sleep.evalkit --manifest tasks.json --a cond_a.json --b cond_b.json +``` + +See [`evalkit.md`](evalkit.md) for the id-set contract, multi-seed bands, the +A/A calibration, and the published RESULTS cell replay. + ## Results > 📊 **More results & analysis — the gate-safety stress test, experience-replay diff --git a/docs/sleep/evalkit.md b/docs/sleep/evalkit.md new file mode 100644 index 00000000..ce6e9c61 --- /dev/null +++ b/docs/sleep/evalkit.md @@ -0,0 +1,53 @@ +# Paired A/B evalkit + +Sleep contributors have a shared instrument for "condition B beats condition A": + +```text +python -m skillopt_sleep.evalkit --manifest tasks.json --a cond_a.json --b cond_b.json +``` + +The kit pairs outcomes by task id, runs McNemar's test on binary successes, and +reports a percentile-bootstrap confidence interval on the success-rate delta. +It does not change the nightly gate. + +## Inputs + +- `--manifest`: JSON list of task ids, or `{"ids": [...]}` / `{"tasks": [{"id": ...}]}`. +- `--a` / `--b`: JSON objects mapping those same ids to `0`/`1` (or a list of + per-seed `0`/`1` values). A wrapper `{"outcomes": {...}}` is also accepted. +- `--aa`: A/A calibration (reuses `--a` as both conditions). Must not reject. +- `--allow-graded`: permit non-binary scores. McNemar is omitted; bootstrap only. +- `--boot`, `--seed`, `--alpha`, `--json`. + +The id sets of the manifest, A, and B must be identical. Cross-manifest +comparisons are refused. + +## Multi-seed + +When each task maps to a same-length list of seed repeats, the kit: + +1. averages per task across seeds for the headline delta and bootstrap CI +2. pools `(task, seed)` pairs for McNemar +3. publishes the per-seed deltas plus their mean and sample sd + +That is the house answer to single-seed noise (see issue #108 and the +single-seed warning in `RESULTS.md`). + +## RESULTS cell replay + +`tests/fixtures/evalkit/results_searchqa_nano_gated.json` replays the published +SearchQA / GPT-5.4-nano / gated / cumulative nights=5 cell (baseline 0.560, +after 0.679, Δ +11.9 on n=1400). Per-task pairs were not published, so the +replay uses a documented maximum-concordance reconstruction: the first +`round(n * rate)` tasks succeed in each condition. The harness recovers the +published delta; it does not claim to recover the original microdata. + +## A/A check + +```text +python -m skillopt_sleep.evalkit --manifest tests/fixtures/evalkit/aa_manifest.json \ + --a tests/fixtures/evalkit/aa_outcomes.json --aa +``` + +Identical conditions must report delta 0, McNemar p_exact = 1, and a CI that +includes 0. If they do not, the statistics are miscoded. diff --git a/skillopt_sleep/__main__.py b/skillopt_sleep/__main__.py index 6875ad21..259de9d4 100644 --- a/skillopt_sleep/__main__.py +++ b/skillopt_sleep/__main__.py @@ -882,6 +882,19 @@ def main(argv=None) -> int: p_unsched = sub.add_parser("unschedule", help="remove the nightly cron entry") _add_common(p_unsched) p_unsched.add_argument("--all", action="store_true", help="remove all managed entries") + p_eval = sub.add_parser( + "evalkit", + help="paired A/B comparison (McNemar + bootstrap CI)", + ) + p_eval.add_argument("--manifest", required=True) + p_eval.add_argument("--a", required=True) + p_eval.add_argument("--b", default="") + p_eval.add_argument("--aa", action="store_true") + p_eval.add_argument("--alpha", type=float, default=0.05) + p_eval.add_argument("--boot", type=int, default=10000) + p_eval.add_argument("--seed", type=int, default=42) + p_eval.add_argument("--allow-graded", action="store_true") + p_eval.add_argument("--json", action="store_true") args = parser.parse_args(argv) if args.cmd == "run": @@ -898,6 +911,19 @@ def main(argv=None) -> int: return cmd_schedule(args) if args.cmd == "unschedule": return cmd_unschedule(args) + if args.cmd == "evalkit": + from skillopt_sleep.evalkit import main as evalkit_main + argv = ["--manifest", args.manifest, "--a", args.a] + if args.b: + argv.extend(["--b", args.b]) + if args.aa: + argv.append("--aa") + argv.extend(["--alpha", str(args.alpha), "--boot", str(args.boot), "--seed", str(args.seed)]) + if args.allow_graded: + argv.append("--allow-graded") + if args.json: + argv.append("--json") + return evalkit_main(argv) parser.print_help() return 2 diff --git a/skillopt_sleep/evalkit.py b/skillopt_sleep/evalkit.py new file mode 100644 index 00000000..8a09dd72 --- /dev/null +++ b/skillopt_sleep/evalkit.py @@ -0,0 +1,479 @@ +"""Paired A/B evaluation kit for SkillOpt-Sleep. + +Sleep reports (and many PRs) quote single-run success rates with no +uncertainty and no guarantee that the two conditions saw the same tasks. +This module is the shared instrument for those comparisons: + + * one fixed task manifest, paired by task id + * McNemar's test on per-task binary outcomes + * percentile-bootstrap confidence intervals on the success-rate delta + * optional multi-seed repeats (per-seed deltas + a pooled pair test) + +It does not change the nightly gate. It standardizes the evidence that +reports and PRs cite. Pure stdlib; no numpy / scipy. + +Refuse comparisons whose task-id sets differ. Graded (non-binary) scores +are bootstrap-only: McNemar is not defined for them. + +CLI:: + + python -m skillopt_sleep.evalkit --manifest M.json --a A.json --b B.json +""" +from __future__ import annotations + +import argparse +import json +import math +import random +import sys +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple + + +# ── errors ──────────────────────────────────────────────────────────────────── + +class EvalkitError(ValueError): + """User-facing contract failure (mismatched ids, empty, etc.).""" + + +# ── results ─────────────────────────────────────────────────────────────────── + +@dataclass +class McNemarResult: + both_success: int + a_only: int # A success, B fail (c in the usual 2x2) + b_only: int # A fail, B success (b) + both_fail: int + n: int + chi2: float # uncorrected (b-c)^2 / (b+c); nan if no discordants + p_chi2: float + p_exact: float # two-sided exact binomial on discordants + significant: bool + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + +@dataclass +class BootstrapCI: + n_boot: int + seed: int + alpha: float + low: float + high: float + mean: float + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + +@dataclass +class EvalReport: + n_tasks: int + rate_a: float + rate_b: float + delta: float + mcnemar: Optional[McNemarResult] + bootstrap: BootstrapCI + per_seed: List[Dict[str, float]] = field(default_factory=list) + seed_mean_delta: Optional[float] = None + seed_sd_delta: Optional[float] = None + notes: List[str] = field(default_factory=list) + refused: bool = False + refuse_reason: str = "" + + def to_dict(self) -> Dict[str, Any]: + d = asdict(self) + if self.mcnemar is not None: + d["mcnemar"] = self.mcnemar.to_dict() + d["bootstrap"] = self.bootstrap.to_dict() + return d + + +# ── statistics ──────────────────────────────────────────────────────────────── + +def _chi2_sf_df1(x: float) -> float: + """Survival function of chi-square with 1 df: P(X > x) = erfc(sqrt(x/2)).""" + if x < 0.0 or math.isnan(x): + return float("nan") + if x == 0.0: + return 1.0 + return math.erfc(math.sqrt(x / 2.0)) + + +def _binom_pmf(k: int, n: int, p: float = 0.5) -> float: + if k < 0 or k > n: + return 0.0 + # nCk * p^k * (1-p)^(n-k). For p=0.5 this is nCk / 2^n. + if p == 0.5: + return math.comb(n, k) / float(1 << n) if n < 1024 else math.comb(n, k) * (0.5 ** n) + return math.comb(n, k) * (p ** k) * ((1.0 - p) ** (n - k)) + + +def exact_mcnemar_p(b: int, c: int) -> float: + """Two-sided exact McNemar p-value (binomial test of discordants, p=0.5).""" + n = b + c + if n == 0: + return 1.0 + k = min(b, c) + tail = sum(_binom_pmf(i, n, 0.5) for i in range(0, k + 1)) + return min(1.0, 2.0 * tail) + + +def mcnemar_from_counts( + both_success: int, + a_only: int, + b_only: int, + both_fail: int, + *, + alpha: float = 0.05, +) -> McNemarResult: + n = both_success + a_only + b_only + both_fail + disc = a_only + b_only + if disc == 0: + chi2 = 0.0 + p_chi2 = 1.0 + else: + chi2 = (b_only - a_only) ** 2 / float(disc) + p_chi2 = _chi2_sf_df1(chi2) + p_exact = exact_mcnemar_p(b_only, a_only) + return McNemarResult( + both_success=both_success, + a_only=a_only, + b_only=b_only, + both_fail=both_fail, + n=n, + chi2=chi2, + p_chi2=p_chi2, + p_exact=p_exact, + significant=p_exact < alpha, + ) + + +def mcnemar_paired(a: Sequence[int], b: Sequence[int], *, alpha: float = 0.05) -> McNemarResult: + if len(a) != len(b): + raise EvalkitError("McNemar requires equal-length paired outcomes") + bs = ao = bo = bf = 0 + for x, y in zip(a, b): + if x and y: + bs += 1 + elif x and not y: + ao += 1 + elif (not x) and y: + bo += 1 + else: + bf += 1 + return mcnemar_from_counts(bs, ao, bo, bf, alpha=alpha) + + +def bootstrap_delta_ci( + a: Sequence[float], + b: Sequence[float], + *, + n_boot: int = 10000, + seed: int = 42, + alpha: float = 0.05, +) -> BootstrapCI: + if len(a) != len(b) or not a: + raise EvalkitError("bootstrap requires a non-empty paired sample") + if n_boot < 1: + raise EvalkitError("n_boot must be >= 1") + rng = random.Random(seed) + n = len(a) + deltas: List[float] = [] + for _ in range(n_boot): + idx = [rng.randrange(n) for _ in range(n)] + da = sum(a[i] for i in idx) / n + db = sum(b[i] for i in idx) / n + deltas.append(db - da) + deltas.sort() + # Inclusive percentile on the sorted sample. + lo_i = int(math.floor((alpha / 2.0) * (n_boot - 1))) + hi_i = int(math.ceil((1.0 - alpha / 2.0) * (n_boot - 1))) + lo_i = max(0, min(n_boot - 1, lo_i)) + hi_i = max(0, min(n_boot - 1, hi_i)) + return BootstrapCI( + n_boot=n_boot, + seed=seed, + alpha=alpha, + low=deltas[lo_i], + high=deltas[hi_i], + mean=sum(deltas) / n_boot, + ) + + +# ── pairing / loading ───────────────────────────────────────────────────────── + +def _as_binary(value: Any) -> Optional[int]: + if value is True or value == 1 or value == 1.0: + return 1 + if value is False or value == 0 or value == 0.0: + return 0 + return None + + +def _normalize_outcomes(raw: Mapping[str, Any]) -> Dict[str, List[float]]: + """Map task id -> list of per-seed scores (length 1 if unseeded).""" + out: Dict[str, List[float]] = {} + for tid, val in raw.items(): + key = str(tid) + if isinstance(val, Mapping) and "seeds" in val: + val = val["seeds"] + if isinstance(val, (list, tuple)): + out[key] = [float(x) for x in val] + else: + out[key] = [float(val)] + return out + + +def align_pairs( + manifest_ids: Sequence[str], + outcomes_a: Mapping[str, Any], + outcomes_b: Mapping[str, Any], +) -> Tuple[List[str], List[List[float]], List[List[float]]]: + """Align A and B onto the manifest. Refuse any id-set mismatch.""" + ids = [str(i) for i in manifest_ids] + if not ids: + raise EvalkitError("manifest is empty") + if len(ids) != len(set(ids)): + raise EvalkitError("manifest has duplicate task ids") + a = _normalize_outcomes(outcomes_a) + b = _normalize_outcomes(outcomes_b) + a_ids, b_ids = set(a), set(b) + want = set(ids) + if a_ids != want or b_ids != want: + missing_a = sorted(want - a_ids) + missing_b = sorted(want - b_ids) + extra_a = sorted(a_ids - want) + extra_b = sorted(b_ids - want) + raise EvalkitError( + "outcome task ids must equal the manifest " + f"(missing_a={missing_a[:8]}, missing_b={missing_b[:8]}, " + f"extra_a={extra_a[:8]}, extra_b={extra_b[:8]})" + ) + n_seed_a = {len(a[i]) for i in ids} + n_seed_b = {len(b[i]) for i in ids} + if len(n_seed_a) != 1 or n_seed_a != n_seed_b: + raise EvalkitError("every task must have the same number of seed repeats in A and B") + return ids, [a[i] for i in ids], [b[i] for i in ids] + + +def _is_binary_matrix(rows: Sequence[Sequence[float]]) -> bool: + for row in rows: + for x in row: + if _as_binary(x) is None: + return False + return True + + +def _mean(xs: Iterable[float]) -> float: + seq = list(xs) + return sum(seq) / len(seq) if seq else float("nan") + + +def _sd(xs: Sequence[float]) -> float: + if len(xs) < 2: + return 0.0 + m = _mean(xs) + return math.sqrt(sum((x - m) ** 2 for x in xs) / (len(xs) - 1)) + + +def reconstruct_paired_from_rates(n: int, rate_a: float, rate_b: float) -> Tuple[List[int], List[int]]: + """Deterministic maximum-concordance reconstruction of paired binaries. + + First ``round(n * rate)`` tasks succeed in each condition, same id order. + This is a published-rate replay convention, not original microdata. + """ + if n < 1: + raise EvalkitError("n must be >= 1") + ka = int(round(n * rate_a)) + kb = int(round(n * rate_b)) + a = [1 if i < ka else 0 for i in range(n)] + b = [1 if i < kb else 0 for i in range(n)] + return a, b + + +def compare( + manifest_ids: Sequence[str], + outcomes_a: Mapping[str, Any], + outcomes_b: Mapping[str, Any], + *, + alpha: float = 0.05, + n_boot: int = 10000, + seed: int = 42, + allow_graded: bool = False, +) -> EvalReport: + ids, a_rows, b_rows = align_pairs(manifest_ids, outcomes_a, outcomes_b) + n_seed = len(a_rows[0]) + notes: List[str] = [] + + # Per-task mean across seeds (the headline paired sample). + a_mean = [_mean(row) for row in a_rows] + b_mean = [_mean(row) for row in b_rows] + rate_a = _mean(a_mean) + rate_b = _mean(b_mean) + delta = rate_b - rate_a + boot = bootstrap_delta_ci(a_mean, b_mean, n_boot=n_boot, seed=seed, alpha=alpha) + + binary = _is_binary_matrix(a_rows) and _is_binary_matrix(b_rows) + mcnemar: Optional[McNemarResult] = None + if binary: + # Pool (task, seed) as paired observations when seeds align. + flat_a = [int(_as_binary(x) or 0) for row in a_rows for x in row] + flat_b = [int(_as_binary(x) or 0) for row in b_rows for x in row] + mcnemar = mcnemar_paired(flat_a, flat_b, alpha=alpha) + elif allow_graded: + notes.append("graded scores: McNemar omitted; bootstrap CI only") + else: + raise EvalkitError( + "non-binary scores require --allow-graded (McNemar is undefined)" + ) + + per_seed: List[Dict[str, float]] = [] + seed_mean = seed_sd = None + if n_seed > 1: + for s in range(n_seed): + da = _mean(row[s] for row in a_rows) + db = _mean(row[s] for row in b_rows) + per_seed.append({"seed": float(s), "rate_a": da, "rate_b": db, "delta": db - da}) + deltas = [row["delta"] for row in per_seed] + seed_mean = _mean(deltas) + seed_sd = _sd(deltas) + notes.append( + f"multi-seed: {n_seed} repeats; seed-mean delta={seed_mean:.6f} " + f"sd={seed_sd:.6f}" + ) + + return EvalReport( + n_tasks=len(ids), + rate_a=rate_a, + rate_b=rate_b, + delta=delta, + mcnemar=mcnemar, + bootstrap=boot, + per_seed=per_seed, + seed_mean_delta=seed_mean, + seed_sd_delta=seed_sd, + notes=notes, + ) + + +def compare_aa( + manifest_ids: Sequence[str], + outcomes: Mapping[str, Any], + **kwargs: Any, +) -> EvalReport: + """A/A calibration: identical conditions must not reject at alpha.""" + report = compare(manifest_ids, outcomes, outcomes, **kwargs) + report.notes.append("A/A calibration (identical conditions)") + return report + + +# ── I/O ─────────────────────────────────────────────────────────────────────── + +def _load_json(path: str) -> Any: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _manifest_ids(obj: Any) -> List[str]: + if isinstance(obj, list): + return [str(x) for x in obj] + if isinstance(obj, Mapping): + if "ids" in obj: + return [str(x) for x in obj["ids"]] + if "tasks" in obj: + return [str(t["id"] if isinstance(t, Mapping) else t) for t in obj["tasks"]] + if "outcomes" in obj: + return [str(k) for k in obj["outcomes"]] + raise EvalkitError("manifest must be a list of ids or an object with ids/tasks") + + +def _outcomes(obj: Any) -> Dict[str, Any]: + if isinstance(obj, Mapping) and "outcomes" in obj: + return dict(obj["outcomes"]) + if isinstance(obj, Mapping): + return dict(obj) + raise EvalkitError("outcomes file must be an object mapping task id to score") + + +def format_markdown(report: EvalReport) -> str: + lines = [ + "# Paired A/B evalkit report", + "", + f"- n_tasks: {report.n_tasks}", + f"- rate_a: {report.rate_a:.6f}", + f"- rate_b: {report.rate_b:.6f}", + f"- delta (B-A): {report.delta:+.6f}", + ( + f"- bootstrap {int((1 - report.bootstrap.alpha) * 100)}% CI: " + f"[{report.bootstrap.low:+.6f}, {report.bootstrap.high:+.6f}] " + f"(n_boot={report.bootstrap.n_boot}, seed={report.bootstrap.seed})" + ), + ] + if report.mcnemar is not None: + m = report.mcnemar + lines.append( + f"- McNemar 2x2: both+={m.both_success} a_only={m.a_only} " + f"b_only={m.b_only} both-={m.both_fail}" + ) + lines.append( + f"- McNemar chi2={m.chi2:.4f} p_chi2={m.p_chi2:.6g} " + f"p_exact={m.p_exact:.6g} significant={m.significant}" + ) + if report.seed_mean_delta is not None: + lines.append( + f"- multi-seed mean delta: {report.seed_mean_delta:+.6f} " + f"(sd {report.seed_sd_delta:.6f}, k={len(report.per_seed)})" + ) + for note in report.notes: + lines.append(f"- note: {note}") + return "\n".join(lines) + "\n" + + +def main(argv: Optional[Sequence[str]] = None) -> int: + p = argparse.ArgumentParser( + prog="skillopt_sleep.evalkit", + description="Paired A/B comparison with McNemar and bootstrap CIs", + ) + p.add_argument("--manifest", required=True, help="JSON list of task ids (or {ids,tasks})") + p.add_argument("--a", required=True, help="JSON outcomes for condition A") + p.add_argument("--b", default="", help="JSON outcomes for condition B (omit for A/A)") + p.add_argument("--aa", action="store_true", help="A/A calibration (ignore --b, reuse --a)") + p.add_argument("--alpha", type=float, default=0.05) + p.add_argument("--boot", type=int, default=10000) + p.add_argument("--seed", type=int, default=42) + p.add_argument("--allow-graded", action="store_true") + p.add_argument("--json", action="store_true") + args = p.parse_args(list(argv) if argv is not None else None) + + try: + ids = _manifest_ids(_load_json(args.manifest)) + a = _outcomes(_load_json(args.a)) + if args.aa or not args.b: + report = compare_aa( + ids, a, alpha=args.alpha, n_boot=args.boot, + seed=args.seed, allow_graded=args.allow_graded, + ) + else: + b = _outcomes(_load_json(args.b)) + report = compare( + ids, a, b, alpha=args.alpha, n_boot=args.boot, + seed=args.seed, allow_graded=args.allow_graded, + ) + except EvalkitError as exc: + print(f"ERR_EVALKIT {exc}", file=sys.stderr) + return 2 + except OSError as exc: + print(f"ERR_EVALKIT {exc}", file=sys.stderr) + return 1 + + if args.json: + print(json.dumps(report.to_dict(), indent=2, sort_keys=True)) + else: + print(format_markdown(report), end="") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/evalkit/aa_manifest.json b/tests/fixtures/evalkit/aa_manifest.json new file mode 100644 index 00000000..05448f5b --- /dev/null +++ b/tests/fixtures/evalkit/aa_manifest.json @@ -0,0 +1,8 @@ +{ + "ids": [ + "t00", "t01", "t02", "t03", "t04", "t05", "t06", "t07", "t08", "t09", + "t10", "t11", "t12", "t13", "t14", "t15", "t16", "t17", "t18", "t19", + "t20", "t21", "t22", "t23", "t24", "t25", "t26", "t27", "t28", "t29", + "t30", "t31", "t32", "t33", "t34", "t35", "t36", "t37", "t38", "t39" + ] +} diff --git a/tests/fixtures/evalkit/aa_outcomes.json b/tests/fixtures/evalkit/aa_outcomes.json new file mode 100644 index 00000000..444ad43f --- /dev/null +++ b/tests/fixtures/evalkit/aa_outcomes.json @@ -0,0 +1,9 @@ +{ + "outcomes": { + "t00": 1, "t01": 1, "t02": 1, "t03": 1, "t04": 1, "t05": 1, "t06": 1, "t07": 1, + "t08": 1, "t09": 1, "t10": 1, "t11": 1, "t12": 1, "t13": 1, "t14": 1, "t15": 1, + "t16": 1, "t17": 1, "t18": 1, "t19": 1, "t20": 0, "t21": 0, "t22": 0, "t23": 0, + "t24": 0, "t25": 0, "t26": 0, "t27": 0, "t28": 0, "t29": 0, "t30": 0, "t31": 0, + "t32": 0, "t33": 0, "t34": 0, "t35": 0, "t36": 0, "t37": 0, "t38": 0, "t39": 0 + } +} diff --git a/tests/fixtures/evalkit/mcnemar_textbook.json b/tests/fixtures/evalkit/mcnemar_textbook.json new file mode 100644 index 00000000..f6019ccb --- /dev/null +++ b/tests/fixtures/evalkit/mcnemar_textbook.json @@ -0,0 +1,10 @@ +{ + "name": "textbook-2x2", + "both_success": 40, + "a_only": 2, + "b_only": 12, + "both_fail": 46, + "chi2": 7.142857142857143, + "p_chi2": 0.007526315166457887, + "p_exact": 0.012939453125 +} diff --git a/tests/fixtures/evalkit/results_searchqa_nano_gated.json b/tests/fixtures/evalkit/results_searchqa_nano_gated.json new file mode 100644 index 00000000..5df05905 --- /dev/null +++ b/tests/fixtures/evalkit/results_searchqa_nano_gated.json @@ -0,0 +1,9 @@ +{ + "cell_id": "results-searchqa-nano-gated-cumulative-nights5", + "source": "docs/sleep/RESULTS.md section 2", + "n": 1400, + "baseline": 0.560, + "after": 0.679, + "published_delta": 0.119, + "reconstruction": "maximum-concordance: first round(n*rate) tasks succeed in each condition" +} diff --git a/tests/test_evalkit.py b/tests/test_evalkit.py new file mode 100644 index 00000000..e09c6338 --- /dev/null +++ b/tests/test_evalkit.py @@ -0,0 +1,214 @@ +"""Paired A/B evalkit: known-answer stats, A/A calibration, RESULTS replay.""" +from __future__ import annotations + +import json +import math +import os +import subprocess +import sys +import tempfile +import unittest + +from skillopt_sleep.evalkit import ( + EvalkitError, + bootstrap_delta_ci, + compare, + compare_aa, + exact_mcnemar_p, + format_markdown, + main as evalkit_main, + mcnemar_from_counts, + mcnemar_paired, + reconstruct_paired_from_rates, +) + + +FIXTURE_DIR = os.path.join(os.path.dirname(__file__), "fixtures", "evalkit") + + +def _load(name: str): + with open(os.path.join(FIXTURE_DIR, name), encoding="utf-8") as f: + return json.load(f) + + +class TestMcNemarKnownAnswer(unittest.TestCase): + def test_textbook_2x2_chi2_and_exact(self): + fx = _load("mcnemar_textbook.json") + res = mcnemar_from_counts( + fx["both_success"], fx["a_only"], fx["b_only"], fx["both_fail"], + ) + self.assertAlmostEqual(res.chi2, fx["chi2"], places=12) + self.assertAlmostEqual(res.p_chi2, fx["p_chi2"], places=12) + self.assertAlmostEqual(res.p_exact, fx["p_exact"], places=12) + self.assertTrue(res.significant) + self.assertEqual(res.n, 100) + + def test_zero_discordants_is_not_significant(self): + res = mcnemar_from_counts(20, 0, 0, 5) + self.assertEqual(res.chi2, 0.0) + self.assertEqual(res.p_chi2, 1.0) + self.assertEqual(res.p_exact, 1.0) + self.assertFalse(res.significant) + + def test_paired_vectors_match_counts(self): + a = [1, 1, 1, 0, 0] + b = [1, 0, 1, 1, 0] + res = mcnemar_paired(a, b) + self.assertEqual(res.both_success, 2) + self.assertEqual(res.a_only, 1) + self.assertEqual(res.b_only, 1) + self.assertEqual(res.both_fail, 1) + self.assertAlmostEqual(res.p_exact, exact_mcnemar_p(1, 1)) + + +class TestBootstrapCoverage(unittest.TestCase): + def test_identical_series_ci_collapses_to_zero(self): + a = [1, 0, 1, 0, 1, 0, 1, 0] + ci = bootstrap_delta_ci(a, a, n_boot=2000, seed=7) + self.assertEqual(ci.low, 0.0) + self.assertEqual(ci.high, 0.0) + self.assertEqual(ci.mean, 0.0) + + def test_known_shift_ci_excludes_zero(self): + # A always 0, B always 1: delta = 1 exactly, CI is [1, 1]. + a = [0] * 30 + b = [1] * 30 + ci = bootstrap_delta_ci(a, b, n_boot=1000, seed=1) + self.assertEqual(ci.low, 1.0) + self.assertEqual(ci.high, 1.0) + + def test_seed_is_deterministic(self): + a = [1, 0, 1, 1, 0, 0, 1, 0, 1, 0] + b = [1, 1, 1, 0, 0, 1, 1, 0, 0, 1] + x = bootstrap_delta_ci(a, b, n_boot=500, seed=99) + y = bootstrap_delta_ci(a, b, n_boot=500, seed=99) + self.assertEqual((x.low, x.high, x.mean), (y.low, y.high, y.mean)) + + +class TestAACalibration(unittest.TestCase): + def test_aa_does_not_reject(self): + man = _load("aa_manifest.json") + out = _load("aa_outcomes.json") + report = compare_aa(man["ids"], out["outcomes"], n_boot=2000, seed=42) + self.assertEqual(report.delta, 0.0) + self.assertIsNotNone(report.mcnemar) + self.assertFalse(report.mcnemar.significant) + self.assertEqual(report.mcnemar.p_exact, 1.0) + self.assertLessEqual(report.bootstrap.low, 0.0) + self.assertGreaterEqual(report.bootstrap.high, 0.0) + + +class TestCompareContracts(unittest.TestCase): + def test_mismatched_ids_are_refused(self): + with self.assertRaises(EvalkitError) as ctx: + compare(["t1", "t2"], {"t1": 1, "t2": 0}, {"t1": 1, "t3": 0}) + self.assertIn("must equal the manifest", str(ctx.exception)) + + def test_duplicate_manifest_ids_refused(self): + with self.assertRaises(EvalkitError): + compare(["t1", "t1"], {"t1": 1}, {"t1": 0}) + + def test_empty_manifest_refused(self): + with self.assertRaises(EvalkitError): + compare([], {}, {}) + + def test_graded_refused_without_flag(self): + with self.assertRaises(EvalkitError) as ctx: + compare(["t1", "t2"], {"t1": 0.4, "t2": 0.9}, {"t1": 0.5, "t2": 0.8}) + self.assertIn("allow-graded", str(ctx.exception)) + + def test_graded_bootstrap_only(self): + report = compare( + ["t1", "t2"], + {"t1": 0.4, "t2": 0.9}, + {"t1": 0.5, "t2": 0.8}, + allow_graded=True, + n_boot=500, + seed=3, + ) + self.assertIsNone(report.mcnemar) + self.assertTrue(any("graded" in n for n in report.notes)) + self.assertAlmostEqual(report.delta, 0.0, places=12) + + def test_multi_seed_variance_band(self): + report = compare( + ["t1", "t2"], + {"t1": [1, 0, 1], "t2": [0, 0, 1]}, + {"t1": [1, 1, 1], "t2": [1, 0, 1]}, + n_boot=400, + seed=2, + ) + self.assertEqual(len(report.per_seed), 3) + self.assertIsNotNone(report.seed_mean_delta) + self.assertGreaterEqual(report.seed_sd_delta, 0.0) + self.assertAlmostEqual(report.rate_a, (2 / 3 + 1 / 3) / 2) + self.assertAlmostEqual(report.rate_b, (1.0 + 2 / 3) / 2) + + +class TestResultsCellReplay(unittest.TestCase): + def test_published_searchqa_nano_gated_delta(self): + cell = _load("results_searchqa_nano_gated.json") + a, b = reconstruct_paired_from_rates(cell["n"], cell["baseline"], cell["after"]) + self.assertEqual(len(a), cell["n"]) + self.assertAlmostEqual(sum(a) / cell["n"], cell["baseline"], places=3) + self.assertAlmostEqual(sum(b) / cell["n"], cell["after"], places=3) + ids = [f"q{i:04d}" for i in range(cell["n"])] + report = compare( + ids, + dict(zip(ids, a)), + dict(zip(ids, b)), + n_boot=800, + seed=42, + ) + self.assertAlmostEqual(report.delta, cell["published_delta"], places=3) + self.assertGreater(report.bootstrap.low, 0.0) + self.assertTrue(report.mcnemar.significant) + md = format_markdown(report) + self.assertIn("delta (B-A)", md) + self.assertIn("McNemar", md) + + +class TestCLI(unittest.TestCase): + def test_aa_cli_exit_zero(self): + rc = evalkit_main([ + "--manifest", os.path.join(FIXTURE_DIR, "aa_manifest.json"), + "--a", os.path.join(FIXTURE_DIR, "aa_outcomes.json"), + "--aa", + "--boot", "300", + "--json", + ]) + self.assertEqual(rc, 0) + + def test_mismatch_cli_exit_two(self): + with tempfile.TemporaryDirectory() as td: + man = os.path.join(td, "m.json") + a = os.path.join(td, "a.json") + b = os.path.join(td, "b.json") + with open(man, "w", encoding="utf-8") as f: + json.dump(["t1", "t2"], f) + with open(a, "w", encoding="utf-8") as f: + json.dump({"t1": 1, "t2": 0}, f) + with open(b, "w", encoding="utf-8") as f: + json.dump({"t1": 1, "t3": 0}, f) + rc = evalkit_main(["--manifest", man, "--a", a, "--b", b]) + self.assertEqual(rc, 2) + + def test_module_entrypoint(self): + proc = subprocess.run( + [ + sys.executable, "-m", "skillopt_sleep.evalkit", + "--manifest", os.path.join(FIXTURE_DIR, "aa_manifest.json"), + "--a", os.path.join(FIXTURE_DIR, "aa_outcomes.json"), + "--aa", + "--boot", "200", + ], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("delta (B-A): +0.000000", proc.stdout) + + +if __name__ == "__main__": + unittest.main()